def _loadFile(self):
     try:
         
         f = open(self._fileName, "r")
         s = f.readline()
         
         while len(s)>1:
             tok = s.split(",")
             """
             function to turn string into entity
             """
             if len(tok) == 3:
                 tok[2] = tok[2].split("\n")
                 book = Book(int(tok[0]), tok[1], tok[2][0])
             else:
                 tok[3] = tok[3].split("\n")
                 book = Book(int(tok[0]), tok[1], tok[2], tok[3][0])
             BookRepository.add(self, book)
             s = f.readline()
         
         
     except Exception as e:
         raise RepoError("Error reading input file: " + str(e))
     finally:
         f.close()
Beispiel #2
0
    def json_to_book(self, json_book):

        if "description" in json_book:
            return Book(int(json_book["id"]), json_book["title"],
                        json_book["author"], json_book["description"])
        return Book(int(json_book["id"]), json_book["title"],
                    json_book["author"])
Beispiel #3
0
def crawlerBook(url, imagePath):
    print("now :" + url)
    header = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36' }
    html = requests.get(url, timeout=(30.0, 30.0), headers=header).text
    soup = BeautifulSoup(html, "html.parser")
    book = Book(
        isbn=crawler.getIsbn(soup),
        name=crawler.getName(soup),
        name2=crawler.getName2(soup),
        author=crawler.getAuthor(soup),
        author2=crawler.getAuthor2(soup),
        translator=crawler.getTranslator(soup),
        publisher=crawler.getPublisher(soup),
        publicationDate=crawler.getPublicationDate(soup),
        language=crawler.getLanguage(soup),
        collection=crawler.getCollection(soup),
        specification=crawler.getSpecification(soup),
        publication=crawler.getPublication(soup),
        classification=crawler.getClassification(soup),
        coverImageUrl=crawler.getCoverImageUrl(soup),
        bookIntroduction=crawler.getBookIntroduction(soup),
        authorIntroduction=crawler.getAuthorIntroduction(soup),
        catalog=crawler.getCatalog(soup),
        preface=crawler.getPreface(soup),
        fromWhere="books"
    )
    # save image
    book.bookUrl = url
    if book.coverImageUrl != None:
        if book.isbn != None:
            book.coverImageId = book.isbn + "-" + book.fromWhere + ".jpg"
            imageSaver.saveImageFile(imagePath + book.coverImageId, book.coverImageUrl)

    return book
    def crawling(self):
        self.book = Book()  # Reinitialize Book object for retry
        log.info("Crawling function is call")
        if self.root_url:
            thumb, title, author, chapter_params = self.get_general_info()

            if Book.general_validate(title, author, chapter_params):
                self.book.set_thumb(thumb)
                self.book.set_title(title)
                self.book.set_author(author)
                for chapter_param in chapter_params:
                    chapter = self.get_chapter(chapter_param)
                    if not Book.chapter_validate(chapter):
                        log.error("Chapter: %s is fail crawled", chapter)
                        return False
                    self.book.add_chapter(chapter)

                self.status = True
                return True
            else:
                log.error("No validate general info")
                return False
        else:
            log.error("not define root url")
            return False
def insertBook():
    params = get_param()
    store = BookStore.get(params['storeName'])
    book = Book(store.id, params["bookName"], params["price"])
    book.insert()
    return {
        "message": "Insert done!"
    }
    def create(book_dict):
        """
        Method to create a new Book record given a book dictionary. Does NOT create associated Author or BookCopy
        records.
        :param book_dict: dictionary of book values for a new record.
        :return: a dictionary object of the created book.
        """
        print("BookDao.create()")

        new_book = Book(**book_dict)
        new_book.publish_date = parser.parse(new_book.publish_date)
        db.session.add(new_book)
        db.session.commit()
        print("book_dao.create() ==> Complete")
        return new_book.to_dict()
Beispiel #7
0
def user_buy_book():
    params = get_param()
    book_store_name = params.get('book_store_name', None)
    book_name = params.get('book_name', None)
    user_id = params.get('id', None)
    if book_store_name is None or book_name is None or user_id is None:
        return {
            'result': 'Error!'
        }
    book_store = BookStore.get(book_store_name)
    if book_store is None:
        return {
            'result': 'Book Store is not found!'
        }
    price = Book.get_price_by_store_name_book_name(book_store.storeName, book_name)
    if price is None:
        return {
            'result': 'Book is not found!'
        }
    user = User.get_by_id(user_id)
    if user is None:
        return {
            'result': 'User is not found!'
        }
    user.cashBalance -= price[0]
    book_store.cashBalance += price[0]
    user.update()
    book_store.update()
    purchase = PurchaseHistory(user_id, book_name, book_store_name, price[0], "today")
    purchase.insert()
    return {
        'result': purchage_history_schema.dump(purchase)
    }
Beispiel #8
0
    def get(self):
        # result = memcache.get("index")
        result = False
        if not result:
            logging.info("cache not hit(index)")
            user = users.get_current_user()
            if user:
                greeting = ("%s : <a href=\"%s\">logout</a>" %
                            (user.nickname(), users.create_logout_url("/")))
            else:
                greeting = ("<a href=\"%s\">login</a>" %
                            users.create_login_url("/"))


            template_values = {
                'greeting': greeting,
                'user': user,
                'activities': Activity.all().order('-created_at').fetch(5),
                'books': Book.all().order('-created_at').fetch(1000),
                }
            path = os.path.join(os.path.dirname(__file__), '..', 'view', 'index.html')
            result = template.render(path, template_values)
            # memcache.set("index", result, 600)

        self.response.out.write(result)
    def test_Repo(self):

        self.assertEqual(len(self.__repo), 0)
        self.__repo.add(self.__book)
        self.assertEqual(len(self.__repo), 1)

        with self.assertRaises(RepoError):
            self.__repo.add(self.__book)

        with self.assertRaises(RepoError):
            self.__repo.remove(self.__nobook)

        with self.assertRaises(RepoError):
            self.__repo.update(self.__nobook)

        self.__repo.update(self.__book)
        self.assertEqual( self.__repo.list(), \
                          "ID \t Title \t  Author \t\t\t\t Description \n {},  {} by {},  described as follows: {}\n".\
                          format(self.__id, self.__title,self.__author,self.__description))
        self.__repo.remove(self.__book)
        with self.assertRaises(RepoError):
            self.__repo.list()

        self.__repo.add(self.__book)
        book = Book(45, "ASC", "Vancea", "du-te la cursurile de ASC")
        self.__repo.add(book)
        self.assertEqual(self.__repo.find(45), book)
        self.assertEqual(self.__repo.find(9), None)

        self.assertEqual(self.__repo.exists_book(book), True)
        self.assertNotEqual(len(self.__repo.get_books()), 0)
        self.assertEqual(len(self.__repo.search_id(45)), 1)
        self.assertEqual(len(self.__repo.search_author("van")), 1)
        self.assertEqual(len(self.__repo.search_description("ursuri")), 1)
        self.assertEqual(len(self.__repo.search_title("asc")), 1)
def book1():
    book = Book(book_id=1,
                title='Old Man',
                publish_date='1980',
                subject='Fiction',
                genre='Novel')
    return book
def book2():
    book = Book(book_id=2,
                title='The Left Hand of Darkness',
                publish_date='1975',
                subject='Fiction',
                genre='Science Fiction')
    return book
def get_item():
    """
    renders the item.html file
    """
    with Book() as book:
        books, json_data = book.get_all()
    return render_template('item.html', books=books, json_data=json_data)
Beispiel #13
0
 def createBook(self, bookList):
     author = bookList[0]
     pages = int(bookList[1])
     isbn = bookList[2]
     title = bookList[3]
     publisher = bookList[4]
     year = int(bookList[5])
     return Book(author, pages, isbn, title, publisher, year)
def create_data():
    """
    Creates a new document in the mongodb database.
    """
    json_data = request.get_json()
    with Book() as book:
        book.insertDocument(json_data)
    return make_response(jsonify({'req': ''}), 200)
    def setUp(self):
        unittest.TestCase.setUp(self)
        self.__id = 23
        self.__title = "FP"
        self.__description = "work on your assignments daily"
        self.__author = "Arthur Molnar"
        self.__book = Book(self.__id, self.__title, self.__author,
                           self.__description)

        self.__noid = -85
        self.__notitle = ""
        self.__noauthor = ""
        self.__nobook = Book(self.__noid, self.__notitle, self.__noauthor,
                             self.__description)

        self.__validator = BookValidator()
        self.__repo = BookRepository()
Beispiel #16
0
    def reandAdCreateBook(self) -> Book:
        title = input("Podaj tytuł: ")
        author = input("Autor: ")
        publisher = input("Wydawnictwo: ")
        isbn = input("ISBN: ")
        relaseDate = int(input("Rok wydania: "))
        pages = int(input("Podaj liczbe stron: "))

        return Book(author, pages, isbn, title, publisher, relaseDate)
def update_data():
    """
    Updates a document in the mongodb database.
    """
    json_data = request.get_json()
    _id = json_data['_id']['$oid']
    with Book() as book:
        book.updateDocument(_id, json_data)
    return make_response(jsonify({'req': ''}), 200)
Beispiel #18
0
 def gridbooksCurrentCellChanged(self):
   cell = self.getCurrentCell()
   if cell is False:
     return
   index=cell.RowIndex
   trace.printlog('currentRowIndex:'+str(index))
   dt=self.view.getattr('gridbooks', 'DataSource')
   dr=dt.Rows[index]
   book = Book(dr['Id'], dr['BookName'], dr['BuyPrice'], dr['BuyDate'], dr['Flag'])
   self.setcurrbook(book)
def index():
    """
    renders the homepage.
    """
    with Book() as book:
        books, json_data = book.get_all()
    item = get_item()
    return render_template('index.html',
                           books=books,
                           item=item,
                           json_data=json_data)
Beispiel #20
0
def parse_document_list(document_list):
    """
    Determines the type of documents in the file and adds them to the library
    """
    for i in range(
            0, len(document_list)
    ):  # Iterates through the list via index, in order to make parsing a bit easier
        if ":" not in document_list[
                i]:  # Makes sure we're processing full documents
            doc_type = document_list[i].strip().rstrip()
            doc_key = document_list[i + 1][4:].strip().rstrip()
            author_list = document_list[i + 2][7:].strip().rstrip().split(",")
            if "Book".lower() in doc_type.lower():
                settings.LIBRARY[doc_key.lower()] = Book(
                    doc_key,  # Book Key
                    author_list,  # Adds author
                    document_list[i + 3][7:].strip().rstrip(),  # Adds title
                    document_list[i +
                                  4][11:].strip().rstrip(),  # Adds publisher
                    document_list[i + 5][6:].strip().rstrip(),  # Adds date
                    doc_type.strip().rstrip()  # Add doc type
                )
            elif "Journal".lower() in doc_type.lower():
                settings.LIBRARY[doc_key.lower()] = Journal(
                    doc_key,  # Journal key
                    author_list,  # Adds author
                    document_list[i + 3][7:].strip().rstrip(),  # Adds title
                    document_list[i + 4][9:].strip().rstrip(),  # Adds Journal
                    document_list[i +
                                  5][11:].strip().rstrip(),  # Adds publisher
                    document_list[i + 6][6:].strip().rstrip(),  # Adds date
                    document_list[i + 7][8:].strip().rstrip(),  # Adds Volume
                    document_list[i + 8][8:].strip().rstrip(),  # Adds Number
                    doc_type  # Adds type
                )
            elif "Conference".lower() in doc_type.lower():
                settings.LIBRARY[doc_key.lower()] = Conference(
                    doc_key,  # Conference key
                    author_list,  # Adds authors
                    document_list[i + 3][7:].strip().rstrip(),  # Adds title
                    document_list[i +
                                  4][12:].strip().rstrip(),  # Adds Conference
                    document_list[i + 5][6:].strip().rstrip(),  # Adds date
                    document_list[i +
                                  6][10:].strip().rstrip(),  # Adds location
                    document_list[i + 7][7:].strip().rstrip(),  # Adds pages
                    doc_type  # Adds type
                )
            else:
                print(
                    f"One of the documents in the list is not a recognized type. Document type: {doc_type}; With key: {doc_key}"
                )
        else:
            continue  # Skips the line b/c its not the start of a document
Beispiel #21
0
 def delete(self, key):
     # FIXME: dirty
     if not users.is_current_user_admin():
         return
     book = Book.get_by_key_name("Book_" + key)
     if book:
         for comment in book.comments: comment.delete()
         for stock in book.stocks: stock.delete()
         book.delete()
     Activity(type='delete', book=book).put()
     self.response.out.write("ok")
     return
async def get_book_with_isbn(isbn: str):
    author_dict = {"name": "name1", "book": ["book1", "book2"]}
    author1 = Author(**author_dict)

    book_dict = {
        "name": "mr kishan",
        "isbn": "124",
        "author": author1,
        "year": 1292
    }
    book1 = Book(**book_dict)
    return book1
Beispiel #23
0
    def post(self):
        id = int(self.request.get("id"))
        bookEdit = Book.get_by_id(id)
        if bookEdit:
            bookEdit.isbn = int(self.request.get("isbn"))
            bookEdit.type = self.request.get("type")
            bookEdit.title = self.request.get("title")
            bookEdit.author = self.request.get("author")
            bookEdit.avaliable = int(self.request.get("avaliable"))

            book_mgt.update(bookEdit)

        self.redirect("/seeBooks")
Beispiel #24
0
 def __can_rent(self, bid, cid):
     """
     Verifies if the book and the client exist so that the rent makes sense
     Input: bid  - positive integer
            cid - positive integer
     Output: True if the book which identifies with bid and the client which identifies with cid exist
             False otherwise
     """
     book = Book(bid, None, None, None)
     client = Client(cid, None)
     if self.__book_repo.exists_book(book) and self.__client_repo.exists_client(client):
         return True
     return False
Beispiel #25
0
 def load_library():
     if not os.path.isfile('static/books.csv'):
         raise FileNotFoundError
     else:
         with open('static/books.csv', 'r') as f:
             for item in f:
                 item = item.strip('\n')
                 item = item.split(',')
                 if item[0] == 'book':
                     author = item[1]
                     title = item[2]
                     book = Book(author, title)
                     book.add_copy(int(item[3]))
                     for copy in book.copies:
                         if item[5] == 'y':
                             copy.is_short_term = True
                     for copy in book.copies[:int(item[4])]:
                         copy.is_borrowed = True
                 else:
                     title = item[1]
                     number = item[2]
                     journal = Journal(title, number)
                     if item[3] == 'y':
                         journal.is_borrowed = True
Beispiel #26
0
def to_model(transport):
    return Book(book_id=transport.id,
                title=transport.title,
                synopsis=transport.synopsis,
                isbn10=transport.isbn10,
                isbn13=transport.isbn13,
                language=transport.language,
                publisher=transport.publisher,
                edition=transport.edition,
                paperback_price=transport.paperback_price,
                ebook_price=transport.ebook_price,
                sold_amount=transport.sold_amount,
                current_amount=transport.current_amount,
                category=transport.category,
                created_time=transport.created_time,
                modified_time=transport.modified_time)
Beispiel #27
0
    def get(self):
        user = users.get_current_user()
        id = int(self.request.get("id"))
        book = Book.get_by_id(id)

        if user and not reserve_mgt.retrieve(book.isbn):
            reserve = reserve_mgt.create_empty_reserve()
            reserve.email = users.get_current_user().email()

            number = book.avaliable - 1
            book.avaliable = number
            reserve.isbn = book.isbn

            book_mgt.update(book)
            reserve_mgt.update(reserve)

        self.redirect("/seeBooks")
 def update_book (self, bid, title, author,  description = None):
     """
     Creates a book with the given attributes and updates the book with the same id, 
     already existing in the repository 
     Input: bid -  positive integer (already existing in one of the books from the repository)
            title, author - string
            description - string (default value the empty string)
     
     """
     book = Book(bid, title, author, description)
     self.__book_valid.valid_book(book)
     b = self.__book_repo.update(book)
     
     redo = FunctionCall(self.update_book, bid, title, author, description)
     undo = FunctionCall(self.update_book, bid, b.get_title(), b.get_author(), b.get_description())
     oper = Operations(undo, redo)
     self.__undoService.add(oper)
Beispiel #29
0
    def get(self):
        user = users.get_current_user()

        if user:

            access_link = users.create_logout_url("/")
            id = int(self.request.get("id"))
            book = Book.get_by_id(id)

        template_values = {
            "book": book,
            "titleView": "Ver en detalle",
            "user": user.email(),
            "access_link": access_link
        }

        jinja = jinja2.get_jinja2(app=self.app)
        self.response.write(jinja.render_template("showBook.html", **template_values))
Beispiel #30
0
def get_content(html):
    """Collect content from the page for the specified classes."""
    soup = BeautifulSoup(html, 'html.parser')
    books = []
    for item in soup.find_all('div', class_='bookkitem'):
        books.append(Book(
            item.find('a', class_='bookkitem_name')
                .get_text(strip=True),

            item.find('div', class_='bookkitem_genre')
                .get_text(strip=True)
                .replace('\n', ''),

            item.find('span', class_='bookkitem_author')
                .get_text(strip=True)
                .replace('авторы', '')
                .replace('автор', '')
            if item.find('span', class_='bookkitem_author')
            else 'Автор неизвестен',

            item.find('div', class_='bookkitem_meta_block')
                .get_text(strip=True)
                .replace('Читает', '')
                .replace('Читают', '')
            if 'минут' not in item.find('div', class_='bookkitem_meta_block')
                                  .get_text(strip=True)
                                  .replace('Читает', '')
                                  .replace('Читают', '')
            else 'Исполнитель неизвестен',

            item.find('div', class_='bookkitem_about')
                .get_text(strip=True)
                .replace('\n', '')
                .replace('\r', '')
                .replace('/', '')
                .replace('а́', 'а')
                .replace('и́', 'и')
                .replace('о́', 'о'),

            HOST + item.find('a', class_='bookkitem_cover')
                       .get('href')
        ))

    return books
Beispiel #31
0
 def post(self):
     book = Book.get_by_key_name("Book_" + self.request.get('book'))
     if not book: return
     body = self.request.get('body')
     if not body: return
     comment = Comment(
         book = book,
         body = body
         )
     comment.put()
     Activity(type='comment', book=book).put()
     user = users.get_current_user()
     template_values = {
         'comment': comment,
         'user': user,
         }
     path = os.path.join(os.path.dirname(__file__), '..', 'view', 'comment/index.html')
     result = template.render(path, template_values)
     self.response.out.write(result)
    def setUp(self):

        self.__id_b = 23
        self.__title = "FP"
        self.__description = "work on your assignments daily"
        self.__author = "Arthur Molnar"
        self.__book = Book(self.__id_b, self.__title, self.__author,
                           self.__description)

        self.__id_cl = 3
        self.__name = "Corina"
        self.__client = Client(self.__id_cl, self.__name)

        self.__rid = 4
        self.__bid = 8
        self.__cid = 2
        self.__rdate = "25.11.2018"
        self.__duedate = "2.12.2018"
        self.__returdate = "28.11.2018"
        self.__rental = Rental(self.__rid, self.__bid, self.__cid,
                               self.__rdate, self.__duedate, self.__returdate)

        self.__bvalid = BookValidator()
        self.__cvalid = ClientValidator()
        self.__rvalid = RentalValidator()

        self.__brepo = BookRepository()
        self.__crepo = ClientRepository()
        self.__rrepo = RentalRepository()

        self.__undoserv = UndoService()

        self.__rental_serv = RentalService(
            self.__rrepo,
            self.__rvalid,
            self.__brepo,
            self.__crepo,
            self.__undoserv,
        )
        self.__book_serv = BookService(self.__brepo, self.__bvalid,
                                       self.__undoserv, self.__rental_serv)
        self.__client_serv = ClientService(self.__crepo, self.__cvalid,
                                           self.__undoserv, self.__rental_serv)
 def add_book(self, bid, title, author, description = None):
     """
     Creates a book with the given attributes and adds it in the repository
     Input: bid -  positive integer 
            title, author - string
            description - string (default value the empty string)
     
     """
     book = Book(bid, title, author, description)
     self.__book_valid.valid_book(book)
     self.__book_repo.add(book)
     
     """
     If everything works fine, we try to implement undo and redo command
     """
     redo = FunctionCall(self.add_book, bid, title, author, description)
     undo = FunctionCall(self.remove_book, bid)
     oper = Operations(undo, redo)
     self.__undoService.add(oper)
Beispiel #34
0
 def get_book(self, book_id):
     book_from_db = None
     conn = self.db_connection_pool.getconn()
     try:
         with conn.cursor() as cursor:
             sql = """
                     SELECT {columns}
                     FROM book WHERE id=%s
                 """.format(columns=BookRepository.columns)
             cursor.execute(sql, (book_id, ))
             result = cursor.fetchone()
             if result is not None:
                 book_from_db = Book(result[0], result[1], result[2],
                                     result[3], result[4], result[5],
                                     result[6], result[7], result[8],
                                     result[9], result[10], result[11],
                                     result[12], result[13], result[14])
     finally:
         self.db_connection_pool.putconn(conn)
     return book_from_db
Beispiel #35
0
 def post(self):
     # FIXME: need exception
     if not users.get_current_user():
         return
     book = Book.create_from_isbn(self.request.get('isbn'))
     if not book.title:
         try:
             book.build_from_isbn()
         except CantBuildBook:
             self.redirect('/')
             return
         book.put()
     
     if self.request.get('owner'):
         owner = users.User(email = self.request.get('owner'))
     else:
         owner = users.get_current_user()
     Stock(book=book, owner=owner).put()
     Activity(type='add', book=book).put()
     self.redirect(book.path())
Beispiel #36
0
 def deepCopy(self, other):
     '''
     Function to deepCopy another LibraryRepository to this (self) one
     It copies all the data from another Repository to this one with no references of the objects (so that the states do not
         depend at all)
     :param other: another LibraryRepository
     '''
     self._books = [
         Book(book.getId(), book.getTitle(), book.getDescription(),
              book.getAuthor()) for book in other.getBooks()
     ]
     self._clients = [
         Client(client.getCnp(), client.getName())
         for client in other.getClients()
     ]
     self._loans = [
         Loan(self.searchClient(loan.getClient().getCnp()),
              self.searchBook(loan.getBook().getId()))
         for loan in other.getLoans()
     ]
Beispiel #37
0
    def get(self, key):
        if key == 'add':
            self.post()
            return
        # result = memcache.get("index")
        result = False
        if result:
            self.response.out.write(result)
            return

        user = users.get_current_user()
        if user:
            greeting = ("<a href=\"%s\">logout %s</a>" %
                        (users.create_logout_url("/"), user.nickname()))
        else:
            greeting = ("<a href=\"%s\">login</a>" %
                        users.create_login_url("/"))

        logging.info("cache not hit()")
        book = Book.get_by_key_name("Book_" + key)
        template_values = None
        if book:
            template_values = {
                'user': user,
                'book': book,
                'greeting': greeting,
                }
            path = os.path.join(os.path.dirname(__file__), '..', 'view', 'book/index.html')
            result = template.render(path, template_values)
        else:
            template_values = {
                'user': user,
                'key':  key,
                'greeting': greeting,
                }
            path = os.path.join(os.path.dirname(__file__), '..', 'view', 'book/not_found.html')
            result = template.render(path, template_values)
        # memcache.set("index", result, 600)
        self.response.out.write(result)
Beispiel #38
0
 def put(self, key):
     if not users.get_current_user():
         self.response.out.write("deny")
         return
     book = Book.get_by_key_name("Book_" + key)
     if not book:
         self.response.out.write("ng")
         return
     if self.request.get('description'):
         stock = book.mystock()
         if stock:
             stock.description = self.request.get('description')
             stock.put()
             self.response.out.write(self.request.get('ok'))
             return
         else:
             self.response.out.write('no stock');
             return
     else:
         type = book.lent_or_return()
         Activity(type=type, book=book).put()
     return
Beispiel #39
0
def book_edit(request):
    '图书编辑'

    book_facade = BookFacade()

    #如果是提交信息
    if request.method =="POST":
        book = Book()
        id = func.get_int_param_from_post(request,'id')
        book.authors =  map(lambda x:x.strip(),request.POST.get('authors','').split('/'))
        book.translators = map(lambda x:x.strip(),request.POST.get('translators','').split('/'))
        book.authors_intro = request.POST.get('authors_intro','')
        book.binding = request.POST.get('binding','')
        book.dir = request.POST.get('dir','')
        book.spic = request.POST.get('spic','')
        book.mpic = request.POST.get('mpic','')
        book.bpic = request.POST.get('bpic','')
        book.isbn10 = request.POST.get('isbn10','')
        book.isbn13 = request.POST.get('isbn13','')
        book.pages = request.POST.get('pages','')
        book.price = request.POST.get('price','')
        book.pubdate = request.POST.get('pubdate','')
        book.publisher = request.POST.get('publisher','')
        book.summary = request.POST.get('summary','')
        book.title = request.POST.get('title','')
        book.sub_title = request.POST.get('sub_title','')
        book.tags = map(lambda x:x.strip(),request.POST.get('tags','').split('/'))


        #修改
        if id>0:
            book.id = id
            messages = book.validate()
            if not messages:
                book_facade.update(book)
                return_url = 'book_list'
                return HttpResponseRedirect(return_url)
            else:
                message=''
                if messages:
                    message = messages[0]
                output = {'message' :message}
                return render_to_response('admin/book_edit.html',output)
        else: #插入
            book.id = func.create_new_id()
            messages = book.validate()
            if not messages:
                book_facade.insert(book)
                return_url = 'book_list'
                return HttpResponseRedirect(return_url)
            else:
                message=''
                if messages:
                    message = messages[0]
                output = {'message' :message}
                return render_to_response('admin/book_edit.html',output)

    id = func.get_int_param_from_get(request,'id')
    output = {}

    if id>0:
        book = book_facade.get_data(id)
        output['book'] = book

    return render_to_response('admin/book_edit.html',output)
Beispiel #40
0
class Crawler:
    book = None
    status = None

    def __init__(self, root_url):
        self.root_url = root_url
        self.book = Book()
        self.status = False

    def set_root_url(self, url):
        self.root_url = url

    def crawling(self):
        self.book = Book()  # Reinitialize Book object for retry
        log.info("Crawling function is call")
        if self.root_url:
            thumb, title, author, chapter_params = self.get_general_info()

            if Book.general_validate(title, author, chapter_params):
                self.book.set_thumb(thumb)
                self.book.set_title(title)
                self.book.set_author(author)
                for chapter_param in chapter_params:
                    chapter = self.get_chapter(chapter_param)
                    if not Book.chapter_validate(chapter):
                        log.error("Chapter: %s is fail crawled", chapter)
                        return False
                    self.book.add_chapter(chapter)

                self.status = True
                return True
            else:
                log.error("No validate general info")
                return False
        else:
            log.error("not define root url")
            return False

    def get_general_info(self):
        log.info("get_general_info function is call")
        book_thumb = None
        book_title = None
        book_author = None
        chapter_params = []

        # site_rs = requests.get(self.root_url, verify=False)
        site_rs = self.try_request(self.root_url)

        soup = BeautifulSoup(site_rs.content, 'html.parser')

        for acronym in soup.find_all('acronym'):
            chapter_link = acronym.li.get('onclick')

            if chapter_link:
                chapter_re = re.search('noidung1\(\'((\w|\W)*)\'\)', chapter_link)
                chapter_param = chapter_re.group(1)
                chapter_params.append(chapter_param)

        if chapter_params:
            first_chapter = self.get_chapter(chapter_params[0])
            if first_chapter:
                if first_chapter.book_thumb:
                    book_thumb = first_chapter.book_thumb
                if first_chapter.book_title:
                    book_title = first_chapter.book_title
                if first_chapter.book_author:
                    book_author = first_chapter.book_author

        return book_thumb, book_title, book_author, chapter_params

    def get_chapter(self, chapter_param):
        log.info("get_chapter function is call")
        log.info(chapter_param)

        book_title = None
        book_thumb = None
        book_author = None
        chapter_title = None
        chapter_content = None
        upper_char = None

        url = system_cfg.CHAPTER_URL
        chapter_param = html_tool.decode_param_to_dict(chapter_param)
        site_rs = self.try_request(url, 'POST', data=chapter_param)
        if site_rs:
            content_list = site_rs.content.split('--!!tach_noi_dung!!--', 3)
            if len(content_list) >= 3:

                #####################
                # get book_thumb from css
                #####################
                css_soup = BeautifulSoup(content_list[0], 'html.parser')
                style_tag = css_soup.find('style')
                if style_tag:
                    thumb_re = re.search('background:url\((http://(\w|\W)*)\)', style_tag.string)
                    if thumb_re:
                        book_thumb = thumb_re.group(1)

                #####################
                # get book title
                # get book author
                # get chapter title
                #####################
                desc_soup = BeautifulSoup(content_list[1], 'html.parser')

                book_title_tag = desc_soup.find('span', class_='chuto40')
                if book_title_tag:
                    book_title = book_title_tag.string.strip()

                tuade_tag = desc_soup.find('div', class_='tuade')
                if tuade_tag:
                    chutieude_tags = desc_soup.find_all('span', class_='chutieude')
                    chutieude_list = []
                    for chutieude_tag in chutieude_tags:
                        if chutieude_tag.string and chutieude_tag.string.strip():
                            chutieude_list.append(chutieude_tag.string.strip())
                    if len(chutieude_list) >= 2:
                        book_author = chutieude_list[0]
                        del chutieude_list[0]
                        for chutieude in chutieude_list:
                            if chapter_title:
                                chapter_title = chapter_title + chutieude + " "
                            else:
                                chapter_title = chutieude + " "
                    elif len(chutieude_list) == 1:
                        chapter_title = chutieude_list[0]
                else:

                    tac_gia_tag = desc_soup.find('span', class_='tacgiaphai')
                    if tac_gia_tag:
                        book_author = tac_gia_tag.string.strip()

                    chutieude_tags = desc_soup.find_all('span', class_='chutieude')
                    chutieude_list = []
                    for chutieude_tag in chutieude_tags:
                        if chutieude_tag.text and chutieude_tag.text.strip():
                            chutieude_list.append(chutieude_tag.text.strip())
                    if len(chutieude_list) == 2:
                        chapter_title = chutieude_list[0] + ": " + chutieude_list[1]
                    elif len(chutieude_list) == 1:
                        chapter_title = chutieude_list[0]

                #####################
                # get chapter content( add chapter title to chapter content)
                #####################

                content_soup = BeautifulSoup(content_list[2], 'html.parser')
                if content_soup:
                    chuhoain_tag = content_soup.find(id='chuhoain')
                    if chuhoain_tag:
                        chuinhoa_img = chuhoain_tag.img
                        if chuinhoa_img:
                            chuhoain_src = chuinhoa_img['src']
                            if (isinstance(chuhoain_src, str) or isinstance(chuhoain_src,
                                                                            unicode)) and chuhoain_src.find(
                                    system_cfg.UPPER_CHAR_URL) != -1:
                                chuhoain = chuhoain_src.replace(system_cfg.UPPER_CHAR_URL, '')
                                chuinhoa_img['src'] = system_cfg.UPPER_CHAR_PATH + chuhoain
                                upper_char = chuhoain

                    chapter_content = content_soup.prettify()

                # Add chapter title to chapter_content
                if chapter_title and chapter_content:
                    chapter_content = '<div><h2 align=\'center\'>' + chapter_title + '</h2></div>' + chapter_content

            chapter = Chapter(title=chapter_title, content=chapter_content)
            if book_title:
                chapter.set_book_title(book_title)
            if book_author:
                chapter.set_book_author(book_author)
            if book_thumb:
                chapter.set_book_thumb(book_thumb)
            if upper_char:
                chapter.set_upper_char(upper_char)

            log.info("Crawler chapter: %s", chapter_title)

            return chapter
        else:
            log.error("Can't get content of this chapter")
            return None

    def try_request(self, url, post_type='GET', try_time=0, params=None, data=None):
        try:

            if post_type == 'GET':
                headers = {'User-Agent': system_cfg.USER_AGENT}
                site_rs = requests.get(url=url, params=params, data=data, headers=headers, verify=False)
            else:

                headers = {'User-Agent': system_cfg.USER_AGENT}
                session = requests.Session()
                session.get(url, headers=headers)
                cookies = requests.utils.cookiejar_from_dict(requests.utils.dict_from_cookiejar(session.cookies))

                site_rs = requests.post(url=url, params=params, data=data, headers=headers, cookies=cookies,
                                        verify=False)

            if not site_rs:
                raise requests.exceptions.ConnectionError

            return site_rs
        except requests.exceptions.ConnectionError:
            try_time += 1
            if try_time >= system_cfg.MAX_RETRY_TIME:
                return None
            else:
                time.sleep(system_cfg.WAITING_TIME)
                log.warn("Retry url %s %r time" % (url, try_time))
                return self.try_request(url, post_type, try_time, params, data)
Beispiel #41
0
 def testBook(self):
     '''
     Method to test the getter and the setter of the Book class
     '''
     book = Book(1, "Introduction to algorithms", "The Bible", "Thomas H Cormen")
     self.assertEqual(self.getId(), 1)
     assert book.getTitle() == "Introduction to algorithms"
     assert book.getDescription() == "The Bible"
     assert book.getAuthor() == "Thomas H Cormen"
     book.setAuthor("Cosmin")
     assert book.getAuthor() == "Cosmin"
     book.setTitle("Title")
     assert book.getTitle() == "Title"
     book.setDescription("Descr")
     assert book.getDescription() == "Descr"
Beispiel #42
0
 def __init__(self, root_url):
     self.root_url = root_url
     self.book = Book()
     self.status = False