def setBookAuthors(book): myDB = database.DBConnection() newauthors = 0 newrefs = 0 try: authorlist = getBookAuthors(book['bookid']) for author in authorlist: authtype = author['type'] if authtype in ['primary author', 'main author', 'secondary author']: if author['role'] in ['Author', '—'] and author['work'] == 'all editions': name = formatAuthorName(unaccented(author['name'])) exists = myDB.match('select authorid from authors where authorname=?', (name,)) if exists: authorid = exists['authorid'] else: # try to add new author to database by name name, authorid, new = lazylibrarian.importer.addAuthorNameToDB(name, False, False) if new and authorid: newauthors += 1 if authorid: # suppress duplicates in bookauthors myDB.action('INSERT into bookauthors (AuthorID, BookID) VALUES (?, ?)', (authorid, book['bookid']), suppress='UNIQUE') newrefs += 1 except Exception as e: logger.debug("Error parsing authorlist for %s: %s" % (book['bookname'], str(e))) return newauthors, newrefs
def setBookAuthors(book): myDB = database.DBConnection() newauthors = 0 newrefs = 0 try: authorlist = getBookAuthors(book['bookid']) for author in authorlist: role = '' if 'id' in author: # it's a goodreads data source authorname = author['name'] exists = myDB.match( 'select authorid from authors where authorid=?', (author['id'], )) if 'role' in author: role = author['role'] else: # it's a librarything data source authorname = formatAuthorName(author['name']) exists = myDB.match( 'select authorid from authors where authorname=?', (authorname, )) if 'type' in author: authtype = author['type'] if authtype in [ 'primary author', 'main author', 'secondary author' ]: role = authtype elif author['role'] in [ 'Author', '—' ] and author['work'] == 'all editions': role = 'Author' if exists: authorid = exists['authorid'] else: # try to add new author to database by name authorname, authorid, new = lazylibrarian.importer.addAuthorNameToDB( authorname, False, False) if new and authorid: newauthors += 1 if authorid: myDB.action( 'INSERT into bookauthors (AuthorID, BookID, Role) VALUES (?, ?, ?)', (authorid, book['bookid'], role), suppress='UNIQUE') newrefs += 1 except Exception as e: logger.error("Error parsing authorlist for %s: %s %s" % (book['bookname'], type(e).__name__, str(e))) return newauthors, newrefs
def _findAuthor(self, **kwargs): if 'name' not in kwargs: self.data = 'Missing parameter: name' return authorname = formatAuthorName(kwargs['name']) if lazylibrarian.CONFIG['BOOK_API'] == "GoogleBooks": GB = GoogleBooks(authorname) queue = Queue.Queue() search_api = threading.Thread(target=GB.find_results, name='API-GBRESULTS', args=[authorname, queue]) search_api.start() else: # lazylibrarian.CONFIG['BOOK_API'] == "GoodReads": GR = GoodReads(authorname) queue = Queue.Queue() search_api = threading.Thread(target=GR.find_results, name='API-GRRESULTS', args=[authorname, queue]) search_api.start() search_api.join() self.data = queue.get()
def setBookAuthors(book): myDB = database.DBConnection() newauthors = 0 newrefs = 0 try: authorlist = getBookAuthors(book['bookid']) for author in authorlist: role = '' if 'id' in author: # it's a goodreads data source authorname = author['name'] exists = myDB.match('select authorid from authors where authorid=?', (author['id'],)) if 'role' in author: role = author['role'] else: # it's a librarything data source authorname = formatAuthorName(author['name']) exists = myDB.match('select authorid from authors where authorname=?', (authorname,)) if 'type' in author: authtype = author['type'] if authtype in ['primary author', 'main author', 'secondary author']: role = authtype elif author['role'] in ['Author', '—'] and author['work'] == 'all editions': role = 'Author' if exists: authorid = exists['authorid'] else: # try to add new author to database by name authorname, authorid, new = lazylibrarian.importer.addAuthorNameToDB(authorname, False, False) if new and authorid: newauthors += 1 if authorid: myDB.action('INSERT into bookauthors (AuthorID, BookID, Role) VALUES (?, ?, ?)', (authorid, book['bookid'], role), suppress='UNIQUE') newrefs += 1 except Exception as e: logger.error("Error parsing authorlist for %s: %s %s" % (book['bookname'], type(e).__name__, str(e))) return newauthors, newrefs
def find_author_id(self, refresh=False): author = self.name author = formatAuthorName(unaccented(author)) URL = 'https://www.goodreads.com/api/author_url/' + urllib.quote(author) + '?' + urllib.urlencode(self.params) # googlebooks gives us author names with long form unicode characters if isinstance(author, str) and hasattr(author, "decode"): author = author.decode('utf-8') # make unicode author = unicodedata.normalize('NFC', author) # normalize to short form logger.debug("Searching for author with name: %s" % author) authorlist = [] try: rootxml, in_cache = get_xml_request(URL, useCache=not refresh) except Exception as e: logger.error("%s finding authorid: %s, %s" % (type(e).__name__, URL, str(e))) return authorlist if rootxml is None: logger.debug("Error requesting authorid") return authorlist resultxml = rootxml.getiterator('author') if resultxml is None: logger.warn('No authors found with name: %s' % author) else: # In spite of how this looks, goodreads only returns one result, even if there are multiple matches # we just have to hope we get the right one. eg search for "James Lovelock" returns "James E. Lovelock" # who only has one book listed under googlebooks, the rest are under "James Lovelock" # goodreads has all his books under "James E. Lovelock". Can't come up with a good solution yet. # For now we'll have to let the user handle this by selecting/adding the author manually for author in resultxml: authorid = author.attrib.get("id") authorlist = self.get_author_info(authorid) return authorlist
def search_wishlist(): try: threadname = threading.currentThread().name if "Thread-" in threadname: threading.currentThread().name = "SEARCHWISHLIST" myDB = database.DBConnection() resultlist, wishproviders = IterateOverWishLists() new_books = 0 if not wishproviders: logger.debug('No wishlists are set') scheduleJob(action='Stop', target='search_wishlist') return # No point in continuing # for each item in resultlist, add to database if necessary, and mark as wanted logger.debug('Processing %s item%s in wishlists' % (len(resultlist), plural(len(resultlist)))) for book in resultlist: # we get rss_author, rss_title, maybe rss_isbn, rss_bookid (goodreads bookid) # we can just use bookid if goodreads, or try isbn and name matching on author/title if not # eg NYTimes wishlist if 'E' in book['types']: ebook_status = "Wanted" else: ebook_status = "Skipped" if 'A' in book['types']: audio_status = "Wanted" else: audio_status = "Skipped" if lazylibrarian.CONFIG['BOOK_API'] == "GoodReads" and book['rss_bookid']: cmd = 'select books.Status as Status,AudioStatus,authors.Status as AuthorStatus,' cmd += 'AuthorName,BookName,Requester,AudioRequester from books,authors ' cmd += 'where books.AuthorID = authors.AuthorID and bookid=?' bookmatch = myDB.match(cmd, (book['rss_bookid'],)) if bookmatch: cmd = 'SELECT SeriesName,Status from series,member ' cmd += 'where series.SeriesID=member.SeriesID and member.BookID=?' series = myDB.select(cmd, (book['rss_bookid'],)) reject_series = None for ser in series: if ser['Status'] in ['Paused', 'Ignored']: reject_series = {"Name": ser['SeriesName'], "Status": ser['Status']} break bookname = bookmatch['BookName'] if bookmatch['Status'] in ['Open', 'Wanted', 'Have']: logger.info('Found book %s, already marked %s' % (bookname, bookmatch['Status'])) if bookmatch["Requester"]: # Already on a wishlist if book["dispname"] not in bookmatch["Requester"]: newValueDict = {"Requester": bookmatch["Requester"] + book["dispname"] + ' '} controlValueDict = {"BookID": book['rss_bookid']} myDB.upsert("books", newValueDict, controlValueDict) else: newValueDict = {"Requester": book["dispname"] + ' '} controlValueDict = {"BookID": book['rss_bookid']} myDB.upsert("books", newValueDict, controlValueDict) elif bookmatch['AuthorStatus'] in ['Paused', 'Ignored']: logger.info('Found book %s, but author is %s' % (bookname, bookmatch['AuthorStatus'])) elif reject_series: logger.info('Found book %s, but series "%s" is %s' % (bookname, reject_series['Name'], reject_series['Status'])) elif ebook_status == "Wanted": # skipped/ignored logger.info('Found book %s, marking as "Wanted"' % bookname) controlValueDict = {"BookID": book['rss_bookid']} newValueDict = {"Status": "Wanted"} myDB.upsert("books", newValueDict, controlValueDict) new_books += 1 if bookmatch["Requester"]: # Already on a wishlist if book["dispname"] not in bookmatch["Requester"]: newValueDict = {"Requester": bookmatch["Requester"] + book["dispname"] + ' '} controlValueDict = {"BookID": book['rss_bookid']} myDB.upsert("books", newValueDict, controlValueDict) else: newValueDict = {"Requester": book["dispname"] + ' '} controlValueDict = {"BookID": book['rss_bookid']} myDB.upsert("books", newValueDict, controlValueDict) if bookmatch['AudioStatus'] in ['Open', 'Wanted', 'Have']: logger.info('Found audiobook %s, already marked %s' % (bookname, bookmatch['AudioStatus'])) if bookmatch["AudioRequester"]: # Already on a wishlist if book["dispname"] not in bookmatch["AudioRequester"]: newValueDict = {"AudioRequester": bookmatch["AudioRequester"] + book["dispname"] + ' '} controlValueDict = {"BookID": book['rss_bookid']} myDB.upsert("books", newValueDict, controlValueDict) else: newValueDict = {"AudioRequester": book["dispname"] + ' '} controlValueDict = {"BookID": book['rss_bookid']} myDB.upsert("books", newValueDict, controlValueDict) elif bookmatch['AuthorStatus'] in ['Paused', 'Ignored']: logger.info('Found book %s, but author is %s' % (bookname, bookmatch['AuthorStatus'])) elif reject_series: logger.info('Found book %s, but series "%s" is %s' % (bookname, reject_series['Name'], reject_series['Status'])) elif audio_status == "Wanted": # skipped/ignored logger.info('Found audiobook %s, marking as "Wanted"' % bookname) controlValueDict = {"BookID": book['rss_bookid']} newValueDict = {"AudioStatus": "Wanted"} myDB.upsert("books", newValueDict, controlValueDict) new_books += 1 if bookmatch["AudioRequester"]: # Already on a wishlist if book["dispname"] not in bookmatch["AudioRequester"]: newValueDict = {"AudioRequester": bookmatch["AudioRequester"] + book["dispname"] + ' '} controlValueDict = {"BookID": book['rss_bookid']} myDB.upsert("books", newValueDict, controlValueDict) else: newValueDict = {"AudioRequester": book["dispname"] + ' '} controlValueDict = {"BookID": book['rss_bookid']} myDB.upsert("books", newValueDict, controlValueDict) else: import_book(book['rss_bookid'], ebook_status, audio_status) new_books += 1 newValueDict = {"Requester": book["dispname"] + ' '} controlValueDict = {"BookID": book['rss_bookid']} myDB.upsert("books", newValueDict, controlValueDict) newValueDict = {"AudioRequester": book["dispname"] + ' '} controlValueDict = {"BookID": book['rss_bookid']} myDB.upsert("books", newValueDict, controlValueDict) else: item = {} results = None item['Title'] = book['rss_title'] if book['rss_bookid']: item['BookID'] = book['rss_bookid'] if book['rss_isbn']: item['ISBN'] = book['rss_isbn'] bookmatch = finditem(item, book['rss_author']) if bookmatch: # it's already in the database authorname = bookmatch['AuthorName'] bookname = bookmatch['BookName'] bookid = bookmatch['BookID'] auth_res = myDB.match('SELECT Status from authors WHERE authorname=?', (authorname,)) if auth_res: auth_status = auth_res['Status'] else: auth_status = 'Unknown' cmd = 'SELECT SeriesName,Status from series,member ' cmd += 'where series.SeriesID=member.SeriesID and member.BookID=?' series = myDB.select(cmd, (book['rss_bookid'],)) reject_series = None for ser in series: if ser['Status'] in ['Paused', 'Ignored']: reject_series = {"Name": ser['SeriesName'], "Status": ser['Status']} break if bookmatch['Status'] in ['Open', 'Wanted', 'Have']: logger.info( 'Found book %s by %s, already marked as "%s"' % (bookname, authorname, bookmatch['Status'])) if bookmatch["Requester"]: # Already on a wishlist if book["dispname"] not in bookmatch["Requester"]: newValueDict = {"Requester": bookmatch["Requester"] + book["dispname"] + ' '} controlValueDict = {"BookID": bookid} myDB.upsert("books", newValueDict, controlValueDict) else: newValueDict = {"Requester": book["dispname"] + ' '} controlValueDict = {"BookID": bookid} myDB.upsert("books", newValueDict, controlValueDict) elif auth_status in ['Paused', 'Ignored']: logger.info('Found book %s, but author is "%s"' % (bookname, auth_status)) elif reject_series: logger.info('Found book %s, but series "%s" is %s' % (bookname, reject_series['Name'], reject_series['Status'])) elif ebook_status == 'Wanted': # skipped/ignored logger.info('Found book %s by %s, marking as "Wanted"' % (bookname, authorname)) controlValueDict = {"BookID": bookid} newValueDict = {"Status": "Wanted"} myDB.upsert("books", newValueDict, controlValueDict) new_books += 1 if bookmatch["Requester"]: # Already on a wishlist if book["dispname"] not in bookmatch["Requester"]: newValueDict = {"Requester": bookmatch["Requester"] + book["dispname"] + ' '} controlValueDict = {"BookID": bookid} myDB.upsert("books", newValueDict, controlValueDict) else: newValueDict = {"Requester": book["dispname"] + ' '} controlValueDict = {"BookID": bookid} myDB.upsert("books", newValueDict, controlValueDict) if bookmatch['AudioStatus'] in ['Open', 'Wanted', 'Have']: logger.info( 'Found audiobook %s by %s, already marked as "%s"' % (bookname, authorname, bookmatch['AudioStatus'])) if bookmatch["AudioRequester"]: # Already on a wishlist if book["dispname"] not in bookmatch["AudioRequester"]: newValueDict = {"AudioRequester": bookmatch["AudioRequester"] + book["dispname"] + ' '} controlValueDict = {"BookID": bookid} myDB.upsert("books", newValueDict, controlValueDict) else: newValueDict = {"AudioRequester": book["dispname"] + ' '} controlValueDict = {"BookID": bookid} myDB.upsert("books", newValueDict, controlValueDict) elif auth_status in ['Paused', 'Ignored']: logger.info('Found book %s, but author is "%s"' % (bookname, auth_status)) elif reject_series: logger.info('Found book %s, but series "%s" is %s' % (bookname, reject_series['Name'], reject_series['Status'])) elif audio_status == 'Wanted': # skipped/ignored logger.info('Found audiobook %s by %s, marking as "Wanted"' % (bookname, authorname)) controlValueDict = {"BookID": bookid} newValueDict = {"AudioStatus": "Wanted"} myDB.upsert("books", newValueDict, controlValueDict) new_books += 1 if bookmatch["AudioRequester"]: # Already on a wishlist if book["dispname"] not in bookmatch["AudioRequester"]: newValueDict = {"AudioRequester": bookmatch["AudioRequester"] + book["dispname"] + ' '} controlValueDict = {"BookID": bookid} myDB.upsert("books", newValueDict, controlValueDict) else: newValueDict = {"AudioRequester": book["dispname"] + ' '} controlValueDict = {"BookID": bookid} myDB.upsert("books", newValueDict, controlValueDict) else: # not in database yet if book['rss_isbn']: results = search_for(book['rss_isbn']) if results: result = results[0] # type: dict if result['isbn_fuzz'] > check_int(lazylibrarian.CONFIG['MATCH_RATIO'], 90): logger.info("Found (%s%%) %s: %s" % (result['isbn_fuzz'], result['authorname'], result['bookname'])) import_book(result['bookid'], ebook_status, audio_status) new_books += 1 newValueDict = {"Requester": book["dispname"] + ' '} controlValueDict = {"BookID": result['bookid']} myDB.upsert("books", newValueDict, controlValueDict) newValueDict = {"AudioRequester": book["dispname"] + ' '} myDB.upsert("books", newValueDict, controlValueDict) bookmatch = True if not results: searchterm = "%s <ll> %s" % (item['Title'], formatAuthorName(book['rss_author'])) results = search_for(unaccented(searchterm)) if results: result = results[0] # type: dict if result['author_fuzz'] > check_int(lazylibrarian.CONFIG['MATCH_RATIO'], 90) \ and result['book_fuzz'] > check_int(lazylibrarian.CONFIG['MATCH_RATIO'], 90): logger.info("Found (%s%% %s%%) %s: %s" % (result['author_fuzz'], result['book_fuzz'], result['authorname'], result['bookname'])) import_book(result['bookid'], ebook_status, audio_status) new_books += 1 newValueDict = {"Requester": book["dispname"] + ' '} controlValueDict = {"BookID": result['bookid']} myDB.upsert("books", newValueDict, controlValueDict) newValueDict = {"AudioRequester": book["dispname"] + ' '} myDB.upsert("books", newValueDict, controlValueDict) bookmatch = True if not bookmatch: msg = "Skipping book %s by %s" % (item['Title'], book['rss_author']) if not results: msg += ', No results returned' logger.warn(msg) else: msg += ', No match found' logger.warn(msg) result = results[0] # type: dict msg = "Closest match (%s%% %s%%) %s: %s" % (result['author_fuzz'], result['book_fuzz'], result['authorname'], result['bookname']) logger.warn(msg) if new_books: logger.info("Wishlist marked %s book%s as Wanted" % (new_books, plural(new_books))) except Exception: logger.error('Unhandled exception in search_wishlist: %s' % traceback.format_exc()) finally: threading.currentThread().name = "WEBSERVER"
def search_rss_book(books=None, library=None): """ books is a list of new books to add, or None for backlog search library is "eBook" or "AudioBook" or None to search all book types """ try: threadname = threading.currentThread().name if "Thread-" in threadname: if books is None: threading.currentThread().name = "SEARCHALLRSS" else: threading.currentThread().name = "SEARCHRSS" if not (lazylibrarian.USE_RSS()): logger.warn('RSS search is disabled') scheduleJob(action='Stop', target='search_rss_book') return myDB = database.DBConnection() resultlist, wishproviders = IterateOverWishLists() new_books = 0 if not wishproviders: logger.debug('No wishlists are set') else: # for each item in resultlist, add to database if necessary, and mark as wanted logger.debug('Processing %s item%s in wishlists' % (len(resultlist), plural(len(resultlist)))) for book in resultlist: # we get rss_author, rss_title, maybe rss_isbn, rss_bookid (goodreads bookid) # we can just use bookid if goodreads, or try isbn and name matching on author/title if googlebooks # not sure if anyone would use a goodreads wishlist if not using goodreads interface... if lazylibrarian.CONFIG['BOOK_API'] == "GoodReads" and book['rss_bookid']: bookmatch = myDB.match('select Status,BookName from books where bookid=?', (book['rss_bookid'],)) if bookmatch: bookstatus = bookmatch['Status'] bookname = bookmatch['BookName'] if bookstatus in ['Open', 'Wanted', 'Have']: logger.info(u'Found book %s, already marked as "%s"' % (bookname, bookstatus)) else: # skipped/ignored logger.info(u'Found book %s, marking as "Wanted"' % bookname) controlValueDict = {"BookID": book['rss_bookid']} newValueDict = {"Status": "Wanted"} myDB.upsert("books", newValueDict, controlValueDict) new_books += 1 else: import_book(book['rss_bookid']) new_books += 1 else: item = {} headers = [] item['Title'] = book['rss_title'] if book['rss_bookid']: item['BookID'] = book['rss_bookid'] headers.append('BookID') if book['rss_isbn']: item['ISBN'] = book['rss_isbn'] headers.append('ISBN') bookmatch = finditem(item, book['rss_author'], headers) if bookmatch: # it's already in the database authorname = bookmatch['AuthorName'] bookname = bookmatch['BookName'] bookid = bookmatch['BookID'] bookstatus = bookmatch['Status'] if bookstatus in ['Open', 'Wanted', 'Have']: logger.info( u'Found book %s by %s, already marked as "%s"' % (bookname, authorname, bookstatus)) else: # skipped/ignored logger.info(u'Found book %s by %s, marking as "Wanted"' % (bookname, authorname)) controlValueDict = {"BookID": bookid} newValueDict = {"Status": "Wanted"} myDB.upsert("books", newValueDict, controlValueDict) new_books += 1 else: # not in database yet results = '' if book['rss_isbn']: results = search_for(book['rss_isbn']) if results: result = results[0] # type: dict if result['isbn_fuzz'] > check_int(lazylibrarian.CONFIG['MATCH_RATIO'], 90): logger.info("Found (%s%%) %s: %s" % (result['isbn_fuzz'], result['authorname'], result['bookname'])) import_book(result['bookid']) new_books += 1 bookmatch = True if not results: searchterm = "%s <ll> %s" % (item['Title'], formatAuthorName(book['rss_author'])) results = search_for(unaccented(searchterm)) if results: result = results[0] # type: dict if result['author_fuzz'] > check_int(lazylibrarian.CONFIG['MATCH_RATIO'], 90) \ and result['book_fuzz'] > check_int(lazylibrarian.CONFIG['MATCH_RATIO'], 90): logger.info("Found (%s%% %s%%) %s: %s" % (result['author_fuzz'], result['book_fuzz'], result['authorname'], result['bookname'])) import_book(result['bookid']) new_books += 1 bookmatch = True if not bookmatch: msg = "Skipping book %s by %s" % (item['Title'], book['rss_author']) if not results: msg += ', No results returned' logger.warn(msg) else: msg += ', No match found' logger.warn(msg) msg = "Closest match (%s%% %s%%) %s: %s" % (result['author_fuzz'], result['book_fuzz'], result['authorname'], result['bookname']) logger.warn(msg) if new_books: logger.info("Wishlist marked %s book%s as Wanted" % (new_books, plural(new_books))) searchbooks = [] if books is None: # We are performing a backlog search cmd = 'SELECT BookID, AuthorName, Bookname, BookSub, BookAdded, books.Status, AudioStatus ' cmd += 'from books,authors WHERE (books.Status="Wanted" OR AudioStatus="Wanted") ' cmd += 'and books.AuthorID = authors.AuthorID order by BookAdded desc' results = myDB.select(cmd) for terms in results: searchbooks.append(terms) else: # The user has added a new book for book in books: cmd = 'SELECT BookID, AuthorName, BookName, BookSub, books.Status, AudioStatus ' cmd += 'from books,authors WHERE BookID=? AND books.AuthorID = authors.AuthorID' results = myDB.select(cmd, (book['bookid'],)) for terms in results: searchbooks.append(terms) if len(searchbooks) == 0: logger.debug("SearchRSS - No books to search for") return resultlist, nproviders = IterateOverRSSSites() if not nproviders and not wishproviders: logger.warn('No rss providers are available') return # No point in continuing logger.info('RSS Searching for %i book%s' % (len(searchbooks), plural(len(searchbooks)))) searchlist = [] for searchbook in searchbooks: # searchterm is only used for display purposes searchterm = searchbook['AuthorName'] + ' ' + searchbook['BookName'] if searchbook['BookSub']: searchterm = searchterm + ': ' + searchbook['BookSub'] if library is None or library == 'eBook': if searchbook['Status'] == "Wanted": searchlist.append( {"bookid": searchbook['BookID'], "bookName": searchbook['BookName'], "bookSub": searchbook['BookSub'], "authorName": searchbook['AuthorName'], "library": "eBook", "searchterm": searchterm}) if library is None or library == 'AudioBook': if searchbook['AudioStatus'] == "Wanted": searchlist.append( {"bookid": searchbook['BookID'], "bookName": searchbook['BookName'], "bookSub": searchbook['BookSub'], "authorName": searchbook['AuthorName'], "library": "AudioBook", "searchterm": searchterm}) rss_count = 0 for book in searchlist: if book['library'] == 'AudioBook': searchtype = 'audio' else: searchtype = 'book' found = processResultList(resultlist, book, searchtype, 'rss') # if you can't find the book, try title without any "(extended details, series etc)" if not found and '(' in book['bookName']: # anything to shorten? searchtype = 'short' + searchtype found = processResultList(resultlist, book, searchtype, 'rss') if not found: logger.info("NZB Searches for %s %s returned no results." % (book['library'], book['searchterm'])) if found > True: rss_count += 1 logger.info("RSS Search for Wanted items complete, found %s book%s" % (rss_count, plural(rss_count))) except Exception: logger.error('Unhandled exception in search_rss_book: %s' % traceback.format_exc()) finally: threading.currentThread().name = "WEBSERVER"
def import_CSV(search_dir=None, library='eBook'): """ Find a csv file in the search_dir and process all the books in it, adding authors to the database if not found and marking the books as "Wanted" Optionally delete the file on successful completion """ # noinspection PyBroadException try: if not search_dir: msg = "Alternate Directory not configured" logger.warn(msg) return msg elif not os.path.isdir(search_dir): msg = "Alternate Directory [%s] not found" % search_dir logger.warn(msg) return msg csvFile = csv_file(search_dir, library=library) headers = None myDB = database.DBConnection() bookcount = 0 authcount = 0 skipcount = 0 total = 0 existing = 0 if not csvFile: msg = "No %s CSV file found in %s" % (library, search_dir) logger.warn(msg) return msg else: logger.debug('Reading file %s' % csvFile) csvreader = reader(open(csvFile, 'rU')) for row in csvreader: if csvreader.line_num == 1: # If we are on the first line, create the headers list from the first row headers = row if 'Author' not in headers or 'Title' not in headers: msg = 'Invalid CSV file found %s' % csvFile logger.warn(msg) return msg else: total += 1 item = dict(list(zip(headers, row))) authorname = formatAuthorName(item['Author']) title = makeUnicode(item['Title']) authmatch = myDB.match('SELECT * FROM authors where AuthorName=?', (authorname,)) if authmatch: logger.debug("CSV: Author %s found in database" % authorname) else: logger.debug("CSV: Author %s not found" % authorname) newauthor, authorid, new = addAuthorNameToDB(author=authorname, addbooks=lazylibrarian.CONFIG['NEWAUTHOR_BOOKS']) if len(newauthor) and newauthor != authorname: logger.debug("Preferred authorname changed from [%s] to [%s]" % (authorname, newauthor)) authorname = newauthor if new: authcount += 1 bookmatch = finditem(item, authorname, library=library) result = '' imported = '' if bookmatch: authorname = bookmatch['AuthorName'] bookname = bookmatch['BookName'] bookid = bookmatch['BookID'] if library == 'eBook': bookstatus = bookmatch['Status'] else: bookstatus = bookmatch['AudioStatus'] if bookstatus in ['Open', 'Wanted', 'Have']: existing += 1 logger.info('Found %s %s by %s, already marked as "%s"' % (library, bookname, authorname, bookstatus)) else: # skipped/ignored logger.info('Found %s %s by %s, marking as "Wanted"' % (library, bookname, authorname)) controlValueDict = {"BookID": bookid} if library == 'eBook': newValueDict = {"Status": "Wanted"} else: newValueDict = {"AudioStatus": "Wanted"} myDB.upsert("books", newValueDict, controlValueDict) bookcount += 1 else: searchterm = "%s <ll> %s" % (title, authorname) results = search_for(unaccented(searchterm)) if results: result = results[0] if result['author_fuzz'] >= lazylibrarian.CONFIG['MATCH_RATIO'] \ and result['book_fuzz'] >= lazylibrarian.CONFIG['MATCH_RATIO']: bookmatch = True if not bookmatch: # no match on full searchterm, try splitting out subtitle newtitle, _ = split_title(authorname, title) if newtitle != title: title = newtitle searchterm = "%s <ll> %s" % (title, authorname) results = search_for(unaccented(searchterm)) if results: result = results[0] if result['author_fuzz'] >= lazylibrarian.CONFIG['MATCH_RATIO'] \ and result['book_fuzz'] >= lazylibrarian.CONFIG['MATCH_RATIO']: bookmatch = True if bookmatch: logger.info("Found (%s%% %s%%) %s: %s for %s: %s" % (result['author_fuzz'], result['book_fuzz'], result['authorname'], result['bookname'], authorname, title)) if library == 'eBook': import_book(result['bookid'], ebook="Wanted", wait=True) else: import_book(result['bookid'], audio="Wanted", wait=True) imported = myDB.match('select * from books where BookID=?', (result['bookid'],)) if imported: bookcount += 1 else: bookmatch = False if not bookmatch: msg = "Skipping book %s by %s" % (title, authorname) if not result: msg += ', No results found' logger.warn(msg) elif not imported: msg += ', Failed to import %s' % result['bookid'] logger.warn(msg) else: msg += ', No match found' logger.warn(msg) msg = "Closest match (%s%% %s%%) %s: %s" % (result['author_fuzz'], result['book_fuzz'], result['authorname'], result['bookname']) logger.warn(msg) skipcount += 1 msg = "Found %i %s%s in csv file, %i already existing or wanted" % (total, library, plural(total), existing) logger.info(msg) msg = "Added %i new author%s, marked %i %s%s as 'Wanted', %i %s%s not found" % \ (authcount, plural(authcount), bookcount, library, plural(bookcount), skipcount, plural(skipcount), library) logger.info(msg) if lazylibrarian.CONFIG['DELETE_CSV']: if skipcount == 0: logger.info("Deleting %s on successful completion" % csvFile) try: os.remove(csvFile) except OSError as why: logger.warn('Unable to delete %s: %s' % (csvFile, why.strerror)) else: logger.warn("Not deleting %s as not all books found" % csvFile) if os.path.isdir(csvFile + '.fail'): try: shutil.rmtree(csvFile + '.fail') except Exception as why: logger.warn("Unable to remove %s, %s %s" % (csvFile + '.fail', type(why).__name__, str(why))) try: _ = safe_move(csvFile, csvFile + '.fail') except Exception as e: logger.error("Unable to rename %s, %s %s" % (csvFile, type(e).__name__, str(e))) if not os.access(csvFile, os.R_OK): logger.error("%s is not readable" % csvFile) if not os.access(csvFile, os.W_OK): logger.error("%s is not writeable" % csvFile) parent = os.path.dirname(csvFile) try: with open(os.path.join(parent, 'll_temp'), 'w') as f: f.write('test') os.remove(os.path.join(parent, 'll_temp')) except Exception as why: logger.error("Directory %s is not writeable: %s" % (parent, why)) return msg except Exception: msg = 'Unhandled exception in importCSV: %s' % traceback.format_exc() logger.error(msg) return msg
def addAuthorNameToDB(author=None, refresh=False, addbooks=True): # get authors name in a consistent format, look them up in the database # if not in database, try to import them. # return authorname,new where new=False if author already in db, new=True if added # authorname returned is our preferred name, or empty string if not found or unable to add new = False if not author or len(author) < 2: logger.debug('Invalid Author Name [%s]' % author) return "", "", False author = formatAuthorName(author) myDB = database.DBConnection() # Check if the author exists, and import the author if not, check_exist_author = myDB.match( 'SELECT AuthorID FROM authors where AuthorName=?', (author.replace('"', '""'), )) if not check_exist_author and lazylibrarian.CONFIG['ADD_AUTHOR']: logger.debug('Author %s not found in database, trying to add' % author) # no match for supplied author, but we're allowed to add new ones GR = GoodReads(author) try: author_gr = GR.find_author_id() except Exception as e: logger.warn("%s finding author id for [%s] %s" % (type(e).__name__, author, str(e))) return "", "", False # only try to add if GR data matches found author data if author_gr: authorname = author_gr['authorname'] # authorid = author_gr['authorid'] # "J.R.R. Tolkien" is the same person as "J. R. R. Tolkien" and "J R R Tolkien" match_auth = author.replace('.', ' ') match_auth = ' '.join(match_auth.split()) match_name = authorname.replace('.', ' ') match_name = ' '.join(match_name.split()) match_name = unaccented(match_name) match_auth = unaccented(match_auth) # allow a degree of fuzziness to cater for different accented character handling. # some author names have accents, # filename may have the accented or un-accented version of the character # The currently non-configurable value of fuzziness might need to go in config # We stored GoodReads unmodified author name in # author_gr, so store in LL db under that # fuzz.ratio doesn't lowercase for us match_fuzz = fuzz.ratio(match_auth.lower(), match_name.lower()) if match_fuzz < 90: logger.debug( "Failed to match author [%s] to authorname [%s] fuzz [%d]" % (author, match_name, match_fuzz)) # To save loading hundreds of books by unknown authors at GR or GB, ignore unknown if (author != "Unknown") and (match_fuzz >= 90): # use "intact" name for author that we stored in # GR author_dict, not one of the various mangled versions # otherwise the books appear to be by a different author! author = author_gr['authorname'] authorid = author_gr['authorid'] # this new authorname may already be in the # database, so check again check_exist_author = myDB.match( 'SELECT AuthorID FROM authors where AuthorID=?', (authorid, )) if check_exist_author: logger.debug('Found goodreads authorname %s in database' % author) else: logger.info("Adding new author [%s]" % author) try: addAuthorToDB(authorname=author, refresh=refresh, authorid=authorid, addbooks=addbooks) check_exist_author = myDB.match( 'SELECT AuthorID FROM authors where AuthorID=?', (authorid, )) if check_exist_author: new = True except Exception as e: logger.debug('Failed to add author [%s] to db: %s %s' % (author, type(e).__name__, str(e))) # check author exists in db, either newly loaded or already there if not check_exist_author: logger.debug("Failed to match author [%s] in database" % author) return "", "", False if isinstance(author, str) and hasattr(author, "decode"): author = author.decode(lazylibrarian.SYS_ENCODING) return author, check_exist_author['AuthorID'], new
def GEN(book=None, prov=None, test=False): errmsg = '' provider = "libgen.io" if prov is None: prov = 'GEN' host = lazylibrarian.CONFIG[prov + '_HOST'] if not host.startswith('http'): host = 'http://' + host search = lazylibrarian.CONFIG[prov + '_SEARCH'] if not search or not search.endswith('.php'): search = 'search.php' if 'index.php' not in search and 'search.php' not in search: search = 'search.php' if search[0] == '/': search = search[1:] sterm = makeUnicode(book['searchterm']) page = 1 results = [] next_page = True while next_page: if 'index.php' in search: params = { "s": book['searchterm'], "f_lang": "All", "f_columns": 0, "f_ext": "All" } else: params = { "view": "simple", "open": 0, "phrase": 0, "column": "def", "res": 100, "req": book['searchterm'] } if page > 1: params['page'] = page providerurl = url_fix(host + "/%s" % search) searchURL = providerurl + "?%s" % urlencode(params) next_page = False result, success = fetchURL(searchURL) if not success: # may return 404 if no results, not really an error if '404' in result: logger.debug("No results found from %s for %s" % (provider, sterm)) success = True elif '111' in result: # looks like libgen has ip based access limits logger.error( 'Access forbidden. Please wait a while before trying %s again.' % provider) errmsg = result else: logger.debug(searchURL) logger.debug('Error fetching page data from %s: %s' % (provider, result)) errmsg = result result = False if test: return success if result: logger.debug('Parsing results from <a href="%s">%s</a>' % (searchURL, provider)) try: soup = BeautifulSoup(result, 'html5lib') rows = [] try: table = soup.find_all('table')[ -1] # un-named table, last one in page if table: rows = table.find_all('tr') except IndexError: # no results table in result page rows = [] if len(rows) > 1: # skip table headers rows = rows[1:] for row in rows: author = '' title = '' size = '' extn = '' link = '' td = row.find_all('td') if 'index.php' in search and len(td) > 3: try: author = formatAuthorName(td[0].text) title = td[2].text newsoup = BeautifulSoup(str(td[4]), 'html5lib') data = newsoup.find('a') if data: link = data.get('href') extn = td[4].text.split('(')[0].strip() size = td[4].text.split('(')[1].split(')')[0] size = size.upper() except IndexError as e: logger.debug( 'Error parsing libgen index.php results: %s' % str(e)) elif 'search.php' in search and len(td) > 8: try: author = formatAuthorName(td[1].text) title = td[2].text size = td[7].text.upper() extn = td[8].text link = '' newsoup = BeautifulSoup(str(td[2]), 'html5lib') for res in newsoup.find_all('a'): output = res.get('href') if 'md5' in output: link = output break except IndexError as e: logger.debug( 'Error parsing libgen search.php results; %s' % str(e)) size = size_in_bytes(size) if link and title: if author: title = author.strip() + ' ' + title.strip() if extn: title = title + '.' + extn if link.startswith('http'): url = redirect_url(host, link) else: if "/index.php?" in link: link = 'md5' + link.split('md5')[1] if "/ads.php?" in link: url = url_fix(host + "/" + link) else: url = url_fix(host + "/ads.php?" + link) bookresult, success = fetchURL(url) if not success: logger.debug( 'Error fetching link data from %s: %s' % (provider, bookresult)) logger.debug(url) url = None else: url = None try: new_soup = BeautifulSoup( bookresult, 'html5lib') for link in new_soup.find_all('a'): output = link.get('href') if output: if output.startswith( 'http' ) and '/get.php' in output: url = output break elif '/get.php' in output: url = '/get.php' + output.split( '/get.php')[1] break elif '/download/book' in output: url = '/download/book' + output.split( '/download/book')[1] break if url and not url.startswith('http'): url = url_fix(host + url) else: url = redirect_url(host, url) except Exception as e: logger.error( '%s parsing bookresult for %s: %s' % (type(e).__name__, link, str(e))) url = None if url: results.append({ 'bookid': book['bookid'], 'tor_prov': provider + '/' + search, 'tor_title': title, 'tor_url': url, 'tor_size': str(size), 'tor_type': 'direct', 'priority': lazylibrarian.CONFIG[prov + '_DLPRIORITY'] }) logger.debug('Found %s, Size %s' % (title, size)) next_page = True except Exception as e: logger.error("An error occurred in the %s parser: %s" % (provider, str(e))) logger.debug('%s: %s' % (provider, traceback.format_exc())) page += 1 if 0 < lazylibrarian.CONFIG['MAX_PAGES'] < page: logger.warn( 'Maximum results page search reached, still more results available' ) next_page = False logger.debug("Found %i result%s from %s for %s" % (len(results), plural(len(results)), provider, sterm)) return results, errmsg
def search_wishlist(): try: threadname = threading.currentThread().name if "Thread-" in threadname: threading.currentThread().name = "SEARCHWISHLIST" myDB = database.DBConnection() resultlist, wishproviders = IterateOverWishLists() new_books = 0 if not wishproviders: logger.debug('No wishlists are set') scheduleJob(action='Stop', target='search_wishlist') return # No point in continuing # for each item in resultlist, add to database if necessary, and mark as wanted logger.debug('Processing %s item%s in wishlists' % (len(resultlist), plural(len(resultlist)))) for book in resultlist: # we get rss_author, rss_title, maybe rss_isbn, rss_bookid (goodreads bookid) # we can just use bookid if goodreads, or try isbn and name matching on author/title if not # eg NYTimes wishlist if 'E' in book['types']: ebook_status = "Wanted" else: ebook_status = "Skipped" if 'A' in book['types']: audio_status = "Wanted" else: audio_status = "Skipped" if lazylibrarian.CONFIG['BOOK_API'] == "GoodReads" and book[ 'rss_bookid']: cmd = 'select Status,AudioStatus,BookName,Requester,AudioRequester from books where bookid=?' bookmatch = myDB.match(cmd, (book['rss_bookid'], )) if bookmatch: bookname = bookmatch['BookName'] if bookmatch['Status'] in ['Open', 'Wanted', 'Have']: logger.info('Found book %s, already marked as "%s"' % (bookname, bookmatch['Status'])) if bookmatch["Requester"]: # Already on a wishlist if book["dispname"] not in bookmatch["Requester"]: newValueDict = { "Requester": bookmatch["Requester"] + book["dispname"] + ' ' } controlValueDict = { "BookID": book['rss_bookid'] } myDB.upsert("books", newValueDict, controlValueDict) else: newValueDict = { "Requester": book["dispname"] + ' ' } controlValueDict = {"BookID": book['rss_bookid']} myDB.upsert("books", newValueDict, controlValueDict) elif ebook_status == "Wanted": # skipped/ignored logger.info('Found book %s, marking as "Wanted"' % bookname) controlValueDict = {"BookID": book['rss_bookid']} newValueDict = {"Status": "Wanted"} myDB.upsert("books", newValueDict, controlValueDict) new_books += 1 if bookmatch["Requester"]: # Already on a wishlist if book["dispname"] not in bookmatch["Requester"]: newValueDict = { "Requester": bookmatch["Requester"] + book["dispname"] + ' ' } controlValueDict = { "BookID": book['rss_bookid'] } myDB.upsert("books", newValueDict, controlValueDict) else: newValueDict = { "Requester": book["dispname"] + ' ' } controlValueDict = {"BookID": book['rss_bookid']} myDB.upsert("books", newValueDict, controlValueDict) if bookmatch['AudioStatus'] in ['Open', 'Wanted', 'Have']: logger.info( 'Found audiobook %s, already marked as "%s"' % (bookname, bookmatch['AudioStatus'])) if bookmatch[ "AudioRequester"]: # Already on a wishlist if book["dispname"] not in bookmatch[ "AudioRequester"]: newValueDict = { "AudioRequester": bookmatch["AudioRequester"] + book["dispname"] + ' ' } controlValueDict = { "BookID": book['rss_bookid'] } myDB.upsert("books", newValueDict, controlValueDict) else: newValueDict = { "AudioRequester": book["dispname"] + ' ' } controlValueDict = {"BookID": book['rss_bookid']} myDB.upsert("books", newValueDict, controlValueDict) elif audio_status == "Wanted": # skipped/ignored logger.info('Found audiobook %s, marking as "Wanted"' % bookname) controlValueDict = {"BookID": book['rss_bookid']} newValueDict = {"AudioStatus": "Wanted"} myDB.upsert("books", newValueDict, controlValueDict) new_books += 1 if bookmatch[ "AudioRequester"]: # Already on a wishlist if book["dispname"] not in bookmatch[ "AudioRequester"]: newValueDict = { "AudioRequester": bookmatch["AudioRequester"] + book["dispname"] + ' ' } controlValueDict = { "BookID": book['rss_bookid'] } myDB.upsert("books", newValueDict, controlValueDict) else: newValueDict = { "AudioRequester": book["dispname"] + ' ' } controlValueDict = {"BookID": book['rss_bookid']} myDB.upsert("books", newValueDict, controlValueDict) else: import_book(book['rss_bookid'], ebook_status, audio_status) new_books += 1 newValueDict = {"Requester": book["dispname"] + ' '} controlValueDict = {"BookID": book['rss_bookid']} myDB.upsert("books", newValueDict, controlValueDict) newValueDict = {"AudioRequester": book["dispname"] + ' '} controlValueDict = {"BookID": book['rss_bookid']} myDB.upsert("books", newValueDict, controlValueDict) else: item = {} results = None item['Title'] = book['rss_title'] if book['rss_bookid']: item['BookID'] = book['rss_bookid'] if book['rss_isbn']: item['ISBN'] = book['rss_isbn'] bookmatch = finditem(item, book['rss_author']) if bookmatch: # it's already in the database authorname = bookmatch['AuthorName'] bookname = bookmatch['BookName'] bookid = bookmatch['BookID'] if bookmatch['Status'] in ['Open', 'Wanted', 'Have']: logger.info( 'Found book %s by %s, already marked as "%s"' % (bookname, authorname, bookmatch['Status'])) if bookmatch["Requester"]: # Already on a wishlist if book["dispname"] not in bookmatch["Requester"]: newValueDict = { "Requester": bookmatch["Requester"] + book["dispname"] + ' ' } controlValueDict = {"BookID": bookid} myDB.upsert("books", newValueDict, controlValueDict) else: newValueDict = { "Requester": book["dispname"] + ' ' } controlValueDict = {"BookID": bookid} myDB.upsert("books", newValueDict, controlValueDict) elif ebook_status == 'Wanted': # skipped/ignored logger.info( 'Found book %s by %s, marking as "Wanted"' % (bookname, authorname)) controlValueDict = {"BookID": bookid} newValueDict = {"Status": "Wanted"} myDB.upsert("books", newValueDict, controlValueDict) new_books += 1 if bookmatch["Requester"]: # Already on a wishlist if book["dispname"] not in bookmatch["Requester"]: newValueDict = { "Requester": bookmatch["Requester"] + book["dispname"] + ' ' } controlValueDict = {"BookID": bookid} myDB.upsert("books", newValueDict, controlValueDict) else: newValueDict = { "Requester": book["dispname"] + ' ' } controlValueDict = {"BookID": bookid} myDB.upsert("books", newValueDict, controlValueDict) if bookmatch['AudioStatus'] in ['Open', 'Wanted', 'Have']: logger.info( 'Found audiobook %s by %s, already marked as "%s"' % (bookname, authorname, bookmatch['AudioStatus'])) if bookmatch[ "AudioRequester"]: # Already on a wishlist if book["dispname"] not in bookmatch[ "AudioRequester"]: newValueDict = { "AudioRequester": bookmatch["AudioRequester"] + book["dispname"] + ' ' } controlValueDict = {"BookID": bookid} myDB.upsert("books", newValueDict, controlValueDict) else: newValueDict = { "AudioRequester": book["dispname"] + ' ' } controlValueDict = {"BookID": bookid} myDB.upsert("books", newValueDict, controlValueDict) elif audio_status == 'Wanted': # skipped/ignored logger.info( 'Found audiobook %s by %s, marking as "Wanted"' % (bookname, authorname)) controlValueDict = {"BookID": bookid} newValueDict = {"AudioStatus": "Wanted"} myDB.upsert("books", newValueDict, controlValueDict) new_books += 1 if bookmatch[ "AudioRequester"]: # Already on a wishlist if book["dispname"] not in bookmatch[ "AudioRequester"]: newValueDict = { "AudioRequester": bookmatch["AudioRequester"] + book["dispname"] + ' ' } controlValueDict = {"BookID": bookid} myDB.upsert("books", newValueDict, controlValueDict) else: newValueDict = { "AudioRequester": book["dispname"] + ' ' } controlValueDict = {"BookID": bookid} myDB.upsert("books", newValueDict, controlValueDict) else: # not in database yet if book['rss_isbn']: results = search_for(book['rss_isbn']) if results: result = results[0] # type: dict if result['isbn_fuzz'] > check_int( lazylibrarian.CONFIG['MATCH_RATIO'], 90): logger.info( "Found (%s%%) %s: %s" % (result['isbn_fuzz'], result['authorname'], result['bookname'])) import_book(result['bookid'], ebook_status, audio_status) new_books += 1 newValueDict = { "Requester": book["dispname"] + ' ' } controlValueDict = {"BookID": result['bookid']} myDB.upsert("books", newValueDict, controlValueDict) newValueDict = { "AudioRequester": book["dispname"] + ' ' } myDB.upsert("books", newValueDict, controlValueDict) bookmatch = True if not results: searchterm = "%s <ll> %s" % (item['Title'], formatAuthorName( book['rss_author'])) results = search_for(unaccented(searchterm)) if results: result = results[0] # type: dict if result['author_fuzz'] > check_int(lazylibrarian.CONFIG['MATCH_RATIO'], 90) \ and result['book_fuzz'] > check_int(lazylibrarian.CONFIG['MATCH_RATIO'], 90): logger.info( "Found (%s%% %s%%) %s: %s" % (result['author_fuzz'], result['book_fuzz'], result['authorname'], result['bookname'])) import_book(result['bookid'], ebook_status, audio_status) new_books += 1 newValueDict = { "Requester": book["dispname"] + ' ' } controlValueDict = {"BookID": result['bookid']} myDB.upsert("books", newValueDict, controlValueDict) newValueDict = { "AudioRequester": book["dispname"] + ' ' } myDB.upsert("books", newValueDict, controlValueDict) bookmatch = True if not bookmatch: msg = "Skipping book %s by %s" % (item['Title'], book['rss_author']) if not results: msg += ', No results returned' logger.warn(msg) else: msg += ', No match found' logger.warn(msg) result = results[0] # type: dict msg = "Closest match (%s%% %s%%) %s: %s" % ( result['author_fuzz'], result['book_fuzz'], result['authorname'], result['bookname']) logger.warn(msg) if new_books: logger.info("Wishlist marked %s book%s as Wanted" % (new_books, plural(new_books))) except Exception: logger.error('Unhandled exception in search_wishlist: %s' % traceback.format_exc()) finally: threading.currentThread().name = "WEBSERVER"
def import_CSV(search_dir=None): """ Find a csv file in the search_dir and process all the books in it, adding authors to the database if not found and marking the books as "Wanted" Optionally delete the file on successful completion """ # noinspection PyBroadException try: if not search_dir: msg = "Alternate Directory not configured" logger.warn(msg) return msg elif not os.path.isdir(search_dir): msg = "Alternate Directory [%s] not found" % search_dir logger.warn(msg) return msg csvFile = csv_file(search_dir) headers = None myDB = database.DBConnection() bookcount = 0 authcount = 0 skipcount = 0 total = 0 existing = 0 if not csvFile: msg = "No CSV file found in %s" % search_dir logger.warn(msg) return msg else: logger.debug('Reading file %s' % csvFile) csvreader = reader(open(csvFile, 'rU')) for row in csvreader: if csvreader.line_num == 1: # If we are on the first line, create the headers list from the first row headers = row if 'Author' not in headers or 'Title' not in headers: msg = 'Invalid CSV file found %s' % csvFile logger.warn(msg) return msg else: total += 1 item = dict(list(zip(headers, row))) authorname = formatAuthorName(item['Author']) title = makeUnicode(item['Title']) authmatch = myDB.match( 'SELECT * FROM authors where AuthorName=?', (authorname, )) if authmatch: logger.debug("CSV: Author %s found in database" % authorname) else: logger.debug("CSV: Author %s not found" % authorname) newauthor, authorid, new = addAuthorNameToDB( author=authorname, addbooks=lazylibrarian.CONFIG['NEWAUTHOR_BOOKS']) if len(newauthor) and newauthor != authorname: logger.debug( "Preferred authorname changed from [%s] to [%s]" % (authorname, newauthor)) authorname = newauthor if new: authcount += 1 bookmatch = finditem(item, authorname) result = '' imported = '' if bookmatch: authorname = bookmatch['AuthorName'] bookname = bookmatch['BookName'] bookid = bookmatch['BookID'] bookstatus = bookmatch['Status'] if bookstatus in ['Open', 'Wanted', 'Have']: existing += 1 logger.info( 'Found book %s by %s, already marked as "%s"' % (bookname, authorname, bookstatus)) else: # skipped/ignored logger.info( 'Found book %s by %s, marking as "Wanted"' % (bookname, authorname)) controlValueDict = {"BookID": bookid} newValueDict = {"Status": "Wanted"} myDB.upsert("books", newValueDict, controlValueDict) bookcount += 1 else: searchterm = "%s <ll> %s" % (title, authorname) results = search_for(unaccented(searchterm)) if results: result = results[0] if result['author_fuzz'] >= lazylibrarian.CONFIG['MATCH_RATIO'] \ and result['book_fuzz'] >= lazylibrarian.CONFIG['MATCH_RATIO']: bookmatch = True if not bookmatch: # no match on full searchterm, try splitting out subtitle newtitle, _ = split_title(authorname, title) if newtitle != title: title = newtitle searchterm = "%s <ll> %s" % (title, authorname) results = search_for(unaccented(searchterm)) if results: result = results[0] if result['author_fuzz'] >= lazylibrarian.CONFIG['MATCH_RATIO'] \ and result['book_fuzz'] >= lazylibrarian.CONFIG['MATCH_RATIO']: bookmatch = True if bookmatch: logger.info( "Found (%s%% %s%%) %s: %s for %s: %s" % (result['author_fuzz'], result['book_fuzz'], result['authorname'], result['bookname'], authorname, title)) import_book(result['bookid'], wait=True) imported = myDB.match( 'select * from books where BookID=?', (result['bookid'], )) if imported: bookcount += 1 else: bookmatch = False if not bookmatch: msg = "Skipping book %s by %s" % (title, authorname) if not result: msg += ', No results found' logger.warn(msg) elif not imported: msg += ', Failed to import %s' % result['bookid'] logger.warn(msg) else: msg += ', No match found' logger.warn(msg) msg = "Closest match (%s%% %s%%) %s: %s" % ( result['author_fuzz'], result['book_fuzz'], result['authorname'], result['bookname']) logger.warn(msg) skipcount += 1 msg = "Found %i book%s in csv file, %i already existing or wanted" % ( total, plural(total), existing) logger.info(msg) msg = "Added %i new author%s, marked %i book%s as 'Wanted', %i book%s not found" % \ (authcount, plural(authcount), bookcount, plural(bookcount), skipcount, plural(skipcount)) logger.info(msg) if lazylibrarian.CONFIG['DELETE_CSV']: if skipcount == 0: logger.info("Deleting %s on successful completion" % csvFile) try: os.remove(csvFile) except OSError as why: logger.warn('Unable to delete %s: %s' % (csvFile, why.strerror)) else: logger.warn("Not deleting %s as not all books found" % csvFile) return msg except Exception: msg = 'Unhandled exception in importCSV: %s' % traceback.format_exc() logger.error(msg) return msg
def addAuthorNameToDB(author=None, refresh=False, addbooks=True): # get authors name in a consistent format, look them up in the database # if not in database, try to import them. # return authorname,authorid,new where new=False if author already in db, new=True if added # authorname returned is our preferred name, or empty string if not found or unable to add new = False if not author or len(author) < 2: logger.debug('Invalid Author Name [%s]' % author) return "", "", False author = formatAuthorName(author) myDB = database.DBConnection() # Check if the author exists, and import the author if not, check_exist_author = myDB.match('SELECT AuthorID FROM authors where AuthorName=?', (author,)) # If no exact match, look for a close fuzzy match to handle misspellings, accents if not check_exist_author: match_name = author.lower() res = myDB.action('select AuthorID,AuthorName from authors') for item in res: aname = item['AuthorName'] if aname: match_fuzz = fuzz.ratio(aname.lower(), match_name) if match_fuzz >= 95: logger.debug("Fuzzy match [%s] %s%% for [%s]" % (item['AuthorName'], match_fuzz, author)) check_exist_author = item author = item['AuthorName'] break if not check_exist_author and lazylibrarian.CONFIG['ADD_AUTHOR']: logger.debug('Author %s not found in database, trying to add' % author) # no match for supplied author, but we're allowed to add new ones GR = GoodReads(author) try: author_gr = GR.find_author_id() except Exception as e: logger.warn("%s finding author id for [%s] %s" % (type(e).__name__, author, str(e))) return "", "", False # only try to add if GR data matches found author data if author_gr: authorname = author_gr['authorname'] # authorid = author_gr['authorid'] # "J.R.R. Tolkien" is the same person as "J. R. R. Tolkien" and "J R R Tolkien" match_auth = author.replace('.', ' ') match_auth = ' '.join(match_auth.split()) match_name = authorname.replace('.', ' ') match_name = ' '.join(match_name.split()) match_name = unaccented(match_name) match_auth = unaccented(match_auth) # allow a degree of fuzziness to cater for different accented character handling. # some author names have accents, # filename may have the accented or un-accented version of the character # The currently non-configurable value of fuzziness might need to go in config # We stored GoodReads unmodified author name in # author_gr, so store in LL db under that # fuzz.ratio doesn't lowercase for us match_fuzz = fuzz.ratio(match_auth.lower(), match_name.lower()) if match_fuzz < 90: logger.debug("Failed to match author [%s] to authorname [%s] fuzz [%d]" % (author, match_name, match_fuzz)) # To save loading hundreds of books by unknown authors at GR or GB, ignore unknown if (author != "Unknown") and (match_fuzz >= 90): # use "intact" name for author that we stored in # GR author_dict, not one of the various mangled versions # otherwise the books appear to be by a different author! author = author_gr['authorname'] authorid = author_gr['authorid'] # this new authorname may already be in the # database, so check again check_exist_author = myDB.match('SELECT AuthorID FROM authors where AuthorID=?', (authorid,)) if check_exist_author: logger.debug('Found goodreads authorname %s in database' % author) else: logger.info("Adding new author [%s]" % author) try: addAuthorToDB(authorname=author, refresh=refresh, authorid=authorid, addbooks=addbooks) check_exist_author = myDB.match('SELECT AuthorID FROM authors where AuthorID=?', (authorid,)) if check_exist_author: new = True except Exception as e: logger.error('Failed to add author [%s] to db: %s %s' % (author, type(e).__name__, str(e))) # check author exists in db, either newly loaded or already there if not check_exist_author: logger.debug("Failed to match author [%s] in database" % author) return "", "", False author = makeUnicode(author) return author, check_exist_author['AuthorID'], new
def GEN(book=None, prov=None): errmsg = '' provider = "libgen.io" if prov is None: prov = 'GEN' host = lazylibrarian.CONFIG[prov + '_HOST'] if not host.startswith('http'): host = 'http://' + host search = lazylibrarian.CONFIG[prov + '_SEARCH'] if not search or not search.endswith('.php'): search = 'search.php' if 'index.php' not in search and 'search.php' not in search: search = 'search.php' if search[0] == '/': search = search[1:] page = 1 results = [] next_page = True while next_page: if 'index.php' in search: params = { "s": book['searchterm'], "f_lang": "All", "f_columns": 0, "f_ext": "All" } else: params = { "view": "simple", "open": 0, "phrase": 0, "column": "def", "res": 100, "req": book['searchterm'] } if page > 1: params['page'] = page providerurl = url_fix(host + "/%s" % search) searchURL = providerurl + "?%s" % urllib.urlencode(params) next_page = False result, success = fetchURL(searchURL) if not success: # may return 404 if no results, not really an error if '404' in result: logger.debug(u"No results found from %s for %s" % (provider, book['searchterm'])) elif '111' in result: # looks like libgen has ip based access limits logger.error( 'Access forbidden. Please wait a while before trying %s again.' % provider) errmsg = result else: logger.debug(searchURL) logger.debug('Error fetching page data from %s: %s' % (provider, result)) errmsg = result result = False if result: logger.debug(u'Parsing results from <a href="%s">%s</a>' % (searchURL, provider)) try: soup = BeautifulSoup(result) try: table = soup.findAll('table')[2] # un-named table if table: rows = table.findAll('tr') except IndexError: # no results table in result page rows = [] if 'search.php' in search and len(rows) > 1: rows = rows[1:] for row in rows: author = '' title = '' size = '' extn = '' link = '' td = row.findAll('td') if 'index.php' in search and len(td) > 3: try: res = str( BeautifulStoneSoup( td[0].text, convertEntities=BeautifulStoneSoup. HTML_ENTITIES)) author = formatAuthorName(res) title = str( BeautifulStoneSoup( td[2].text, convertEntities=BeautifulStoneSoup. HTML_ENTITIES)) temp = str(td[4]) temp = temp.split('onmouseout')[1] extn = temp.split('">')[1].split('(')[0] size = temp.split('">')[1].split('(')[1].split( ')')[0] size = size.upper() link = temp.split('href=')[1].split('"')[1] except IndexError as e: logger.debug( 'Error parsing libgen index.php results: %s' % str(e)) elif 'search.php' in search and len(td) > 8: try: res = str( BeautifulStoneSoup( td[1].text, convertEntities=BeautifulStoneSoup. HTML_ENTITIES)) author = formatAuthorName(res) title = str( td[2]).split('>')[2].split('<')[0].strip() title = str( BeautifulStoneSoup( title, convertEntities=BeautifulStoneSoup. HTML_ENTITIES)) link = str(td[2]).split('href="')[1].split( '?')[1].split('"')[0] size = unaccented(td[7].text).upper() extn = td[8].text except IndexError as e: logger.debug( 'Error parsing libgen search.php results; %s' % str(e)) if not size: size = 0 else: try: mult = 1 if 'K' in size: size = size.split('K')[0] mult = 1024 elif 'M' in size: size = size.split('M')[0] mult = 1024 * 1024 elif 'G' in size: size = size.split('G')[0] mult = 1024 * 1024 * 1024 size = int(float(size) * mult) except (ValueError, IndexError): size = 0 if link and title: if author: title = author.strip() + ' ' + title.strip() if extn: title = title + '.' + extn if not link.startswith('http'): if "/ads.php?" in link: url = url_fix(host + link) else: url = url_fix(host + "/ads.php?" + link) else: url = redirect_url(host, link) bookresult, success = fetchURL(url) if not success: # may return 404 if no results, not really an error if '404' in bookresult: logger.debug( u"No results found from %s for %s" % (provider, book['searchterm'])) else: logger.debug(url) logger.debug( 'Error fetching link data from %s: %s' % (provider, bookresult)) errmsg = bookresult bookresult = False if bookresult: url = None try: new_soup = BeautifulSoup(bookresult) for link in new_soup.findAll('a'): output = link.get('href') if output: if output.startswith( 'http' ) and '/get.php' in output: url = output break elif '/get.php' in output: url = '/get.php' + output.split( '/get.php')[1] break elif '/download/book' in output: url = '/download/book' + output.split( '/download/book')[1] break if url and not url.startswith('http'): url = url_fix(host + url) else: url = redirect_url(host, url) except Exception as e: logger.debug( 'Error parsing bookresult for %s: %s' % (link, str(e))) url = None if url: results.append({ 'bookid': book['bookid'], 'tor_prov': provider + '/' + search, 'tor_title': title, 'tor_url': url, 'tor_size': str(size), 'tor_type': 'direct', 'priority': lazylibrarian.CONFIG[prov + '_DLPRIORITY'] }) logger.debug('Found %s, Size %s' % (title, size)) next_page = True except Exception as e: logger.error(u"An error occurred in the %s parser: %s" % (provider, str(e))) logger.debug('%s: %s' % (provider, traceback.format_exc())) page += 1 if 0 < lazylibrarian.CONFIG['MAX_PAGES'] < page: logger.warn( 'Maximum results page search reached, still more results available' ) next_page = False logger.debug( u"Found %i result%s from %s for %s" % (len(results), plural(len(results)), provider, book['searchterm'])) return results, errmsg
def GEN(book=None, prov=None, test=False): errmsg = '' provider = "libgen.io" if not prov: prov = 'GEN' host = lazylibrarian.CONFIG[prov + '_HOST'] if not host.startswith('http'): host = 'http://' + host search = lazylibrarian.CONFIG[prov + '_SEARCH'] if not search or not search.endswith('.php'): search = 'search.php' if 'index.php' not in search and 'search.php' not in search: search = 'search.php' if search[0] == '/': search = search[1:] sterm = makeUnicode(book['searchterm']) page = 1 results = [] next_page = True while next_page: if 'index.php' in search: params = { "s": book['searchterm'], "f_lang": "All", "f_columns": 0, "f_ext": "All" } else: params = { "view": "simple", "open": 0, "phrase": 0, "column": "def", "res": 100, "req": book['searchterm'] } if page > 1: params['page'] = page providerurl = url_fix(host + "/%s" % search) searchURL = providerurl + "?%s" % urlencode(params) next_page = False result, success = fetchURL(searchURL) if not success: # may return 404 if no results, not really an error if '404' in result: logger.debug("No results found from %s for %s" % (provider, sterm)) success = True elif '111' in result: # looks like libgen has ip based access limits logger.error('Access forbidden. Please wait a while before trying %s again.' % provider) errmsg = result else: logger.debug(searchURL) logger.debug('Error fetching page data from %s: %s' % (provider, result)) errmsg = result result = False if test: return success if result: logger.debug('Parsing results from <a href="%s">%s</a>' % (searchURL, provider)) try: soup = BeautifulSoup(result, 'html5lib') rows = [] try: table = soup.find_all('table', rules='rows')[-1] # the last table with rules=rows if table: rows = table.find_all('tr') except IndexError: # no results table in result page rows = [] if len(rows) > 1: # skip table headers rows = rows[1:] for row in rows: author = '' title = '' size = '' extn = '' link = '' td = row.find_all('td') if 'index.php' in search and len(td) > 3: # Foreign fiction try: author = formatAuthorName(td[0].text) title = td[2].text newsoup = BeautifulSoup(str(td[4]), 'html5lib') data = newsoup.find('a') if data: link = data.get('href') extn = td[4].text.split('(')[0].strip() size = td[4].text.split('(')[1].split(')')[0] size = size.upper() except IndexError as e: logger.debug('Error parsing libgen index.php results: %s' % str(e)) elif 'search.php' in search and len(td) > 8: # Non-fiction try: author = formatAuthorName(td[1].text) title = td[2].text size = td[7].text.upper() extn = td[8].text link = '' newsoup = BeautifulSoup(str(td[2]), 'html5lib') for res in newsoup.find_all('a'): output = res.get('href') if 'md5' in output: link = output break except IndexError as e: logger.debug('Error parsing libgen search.php results; %s' % str(e)) size = size_in_bytes(size) if link and title: if author: title = author.strip() + ' ' + title.strip() if extn: title = title + '.' + extn if link.startswith('http'): url = redirect_url(host, link) else: if "/index.php?" in link: link = 'md5' + link.split('md5')[1] if "/ads.php?" in link: url = url_fix(host + "/" + link) else: url = url_fix(host + "/ads.php?" + link) bookresult, success = fetchURL(url) if not success: logger.debug('Error fetching link data from %s: %s' % (provider, bookresult)) logger.debug(url) url = None else: url = None try: new_soup = BeautifulSoup(bookresult, 'html5lib') for link in new_soup.find_all('a'): output = link.get('href') if output: if output.startswith('http') and '/get.php' in output: url = output break elif '/get.php' in output: url = '/get.php' + output.split('/get.php')[1] break elif '/download/book' in output: url = '/download/book' + output.split('/download/book')[1] break if url and not url.startswith('http'): url = url_fix(host + url) else: url = redirect_url(host, url) except Exception as e: logger.error('%s parsing bookresult for %s: %s' % (type(e).__name__, link, str(e))) url = None if url: results.append({ 'bookid': book['bookid'], 'tor_prov': provider + '/' + search, 'tor_title': title, 'tor_url': url, 'tor_size': str(size), 'tor_type': 'direct', 'priority': lazylibrarian.CONFIG[prov + '_DLPRIORITY'] }) logger.debug('Found %s, Size %s' % (title, size)) next_page = True except Exception as e: logger.error("An error occurred in the %s parser: %s" % (provider, str(e))) logger.debug('%s: %s' % (provider, traceback.format_exc())) page += 1 if 0 < lazylibrarian.CONFIG['MAX_PAGES'] < page: logger.warn('Maximum results page search reached, still more results available') next_page = False logger.debug("Found %i result%s from %s for %s" % (len(results), plural(len(results)), provider, sterm)) return results, errmsg
def import_CSV(search_dir=None): """ Find a csv file in the search_dir and process all the books in it, adding authors to the database if not found and marking the books as "Wanted" """ try: if not search_dir: msg = "Alternate Directory not configured" logger.warn(msg) return msg elif not os.path.isdir(search_dir): msg = "Alternate Directory [%s] not found" % search_dir logger.warn(msg) return msg csvFile = csv_file(search_dir) headers = None content = {} if not csvFile: msg = "No CSV file found in %s" % search_dir logger.warn(msg) return msg else: logger.debug(u'Reading file %s' % csvFile) reader = csv.reader(open(csvFile)) for row in reader: if reader.line_num == 1: # If we are on the first line, create the headers list from the first row headers = row else: # Otherwise, the key in the content dictionary is the first item in the # row and we can create the sub-dictionary by using the zip() function. # we include the key in the dictionary as our exported csv files use # bookid as the key content[row[0]] = dict(zip(headers, row)) # We can now get to the content by using the resulting dictionary, so to see # the list of lines, we can do: print content.keys() to get a list of keys # To see the list of fields available for each book: print headers if 'Author' not in headers or 'Title' not in headers: msg = 'Invalid CSV file found %s' % csvFile logger.warn(msg) return msg myDB = database.DBConnection() bookcount = 0 authcount = 0 skipcount = 0 logger.debug(u"CSV: Found %s book%s in csv file" % (len(content.keys()), plural(len(content.keys())))) for item in content.keys(): authorname = content[item]['Author'] if isinstance(authorname, str) and hasattr(authorname, "decode"): authorname = authorname.decode(lazylibrarian.SYS_ENCODING) authorname = formatAuthorName(authorname) title = content[item]['Title'] if isinstance(title, str) and hasattr(title, "decode"): title = title.decode(lazylibrarian.SYS_ENCODING) authmatch = myDB.match('SELECT * FROM authors where AuthorName=?', (authorname,)) if authmatch: logger.debug(u"CSV: Author %s found in database" % authorname) else: logger.debug(u"CSV: Author %s not found" % authorname) newauthor, authorid, new = addAuthorNameToDB(author=authorname, addbooks=lazylibrarian.CONFIG['NEWAUTHOR_BOOKS']) if len(newauthor) and newauthor != authorname: logger.debug("Preferred authorname changed from [%s] to [%s]" % (authorname, newauthor)) authorname = newauthor if new: authcount += 1 bookmatch = finditem(content[item], authorname, headers) result = '' if bookmatch: authorname = bookmatch['AuthorName'] bookname = bookmatch['BookName'] bookid = bookmatch['BookID'] bookstatus = bookmatch['Status'] if bookstatus in ['Open', 'Wanted', 'Have']: logger.info(u'Found book %s by %s, already marked as "%s"' % (bookname, authorname, bookstatus)) else: # skipped/ignored logger.info(u'Found book %s by %s, marking as "Wanted"' % (bookname, authorname)) controlValueDict = {"BookID": bookid} newValueDict = {"Status": "Wanted"} myDB.upsert("books", newValueDict, controlValueDict) bookcount += 1 else: searchterm = "%s <ll> %s" % (title, authorname) results = search_for(unaccented(searchterm)) if results: result = results[0] if result['author_fuzz'] > lazylibrarian.CONFIG['MATCH_RATIO'] \ and result['book_fuzz'] > lazylibrarian.CONFIG['MATCH_RATIO']: logger.info("Found (%s%% %s%%) %s: %s" % (result['author_fuzz'], result['book_fuzz'], result['authorname'], result['bookname'])) import_book(result['bookid']) bookcount += 1 bookmatch = True if not bookmatch: msg = "Skipping book %s by %s" % (title, authorname) if not result: msg += ', No results returned' logger.warn(msg) else: msg += ', No match found' logger.warn(msg) msg = "Closest match (%s%% %s%%) %s: %s" % (result['author_fuzz'], result['book_fuzz'], result['authorname'], result['bookname']) logger.warn(msg) skipcount += 1 msg = "Added %i new author%s, marked %i book%s as 'Wanted', %i book%s not found" % \ (authcount, plural(authcount), bookcount, plural(bookcount), skipcount, plural(skipcount)) logger.info(msg) return msg except Exception: msg = 'Unhandled exception in importCSV: %s' % traceback.format_exc() logger.error(msg) return msg
def search_wishlist(): if not (lazylibrarian.USE_RSS()): logger.warn('RSS search is disabled') scheduleJob(action='Stop', target='search_wishlist') return try: threadname = threading.currentThread().name if "Thread-" in threadname: threading.currentThread().name = "SEARCHWISHLIST" myDB = database.DBConnection() resultlist, wishproviders = IterateOverWishLists() new_books = 0 if not wishproviders: logger.debug('No wishlists are set') scheduleJob(action='Stop', target='search_wishlist') return # No point in continuing # for each item in resultlist, add to database if necessary, and mark as wanted logger.debug('Processing %s item%s in wishlists' % (len(resultlist), plural(len(resultlist)))) for book in resultlist: # we get rss_author, rss_title, maybe rss_isbn, rss_bookid (goodreads bookid) # we can just use bookid if goodreads, or try isbn and name matching on author/title if googlebooks # not sure if anyone would use a goodreads wishlist if not using goodreads interface... if lazylibrarian.CONFIG['BOOK_API'] == "GoodReads" and book['rss_bookid']: bookmatch = myDB.match('select Status,BookName from books where bookid=?', (book['rss_bookid'],)) if bookmatch: bookstatus = bookmatch['Status'] bookname = bookmatch['BookName'] if bookstatus in ['Open', 'Wanted', 'Have']: logger.info('Found book %s, already marked as "%s"' % (bookname, bookstatus)) else: # skipped/ignored logger.info('Found book %s, marking as "Wanted"' % bookname) controlValueDict = {"BookID": book['rss_bookid']} newValueDict = {"Status": "Wanted"} myDB.upsert("books", newValueDict, controlValueDict) new_books += 1 else: import_book(book['rss_bookid']) new_books += 1 else: item = {} results = None item['Title'] = book['rss_title'] if book['rss_bookid']: item['BookID'] = book['rss_bookid'] if book['rss_isbn']: item['ISBN'] = book['rss_isbn'] bookmatch = finditem(item, book['rss_author']) if bookmatch: # it's already in the database authorname = bookmatch['AuthorName'] bookname = bookmatch['BookName'] bookid = bookmatch['BookID'] bookstatus = bookmatch['Status'] if bookstatus in ['Open', 'Wanted', 'Have']: logger.info( 'Found book %s by %s, already marked as "%s"' % (bookname, authorname, bookstatus)) else: # skipped/ignored logger.info('Found book %s by %s, marking as "Wanted"' % (bookname, authorname)) controlValueDict = {"BookID": bookid} newValueDict = {"Status": "Wanted"} myDB.upsert("books", newValueDict, controlValueDict) new_books += 1 else: # not in database yet if book['rss_isbn']: results = search_for(book['rss_isbn']) if results: result = results[0] # type: dict if result['isbn_fuzz'] > check_int(lazylibrarian.CONFIG['MATCH_RATIO'], 90): logger.info("Found (%s%%) %s: %s" % (result['isbn_fuzz'], result['authorname'], result['bookname'])) import_book(result['bookid']) new_books += 1 bookmatch = True if not results: searchterm = "%s <ll> %s" % (item['Title'], formatAuthorName(book['rss_author'])) results = search_for(unaccented(searchterm)) if results: result = results[0] # type: dict if result['author_fuzz'] > check_int(lazylibrarian.CONFIG['MATCH_RATIO'], 90) \ and result['book_fuzz'] > check_int(lazylibrarian.CONFIG['MATCH_RATIO'], 90): logger.info("Found (%s%% %s%%) %s: %s" % (result['author_fuzz'], result['book_fuzz'], result['authorname'], result['bookname'])) import_book(result['bookid']) new_books += 1 bookmatch = True if not bookmatch: msg = "Skipping book %s by %s" % (item['Title'], book['rss_author']) if not results: msg += ', No results returned' logger.warn(msg) else: msg += ', No match found' logger.warn(msg) result = results[0] # type: dict msg = "Closest match (%s%% %s%%) %s: %s" % (result['author_fuzz'], result['book_fuzz'], result['authorname'], result['bookname']) logger.warn(msg) if new_books: logger.info("Wishlist marked %s book%s as Wanted" % (new_books, plural(new_books))) except Exception: logger.error('Unhandled exception in search_wishlist: %s' % traceback.format_exc()) finally: threading.currentThread().name = "WEBSERVER"