def _addAuthor(self, **kwargs): if 'name' not in kwargs: self.data = 'Missing parameter: name' return else: self.id = kwargs['name'] try: self.data = addAuthorNameToDB(author=self.id, refresh=False) except Exception as e: self.data = "%s %s" % (type(e).__name__, str(e))
def import_CSV(search_dir=None): """ Find a csv file in the search_dir and process all the books in it, adding authors to the database if not found and marking the books as "Wanted" """ try: if not search_dir: msg = "Alternate Directory not configured" logger.warn(msg) return msg elif not os.path.isdir(search_dir): msg = "Alternate Directory [%s] not found" % search_dir logger.warn(msg) return msg csvFile = csv_file(search_dir) headers = None content = {} if not csvFile: msg = "No CSV file found in %s" % search_dir logger.warn(msg) return msg else: logger.debug(u'Reading file %s' % csvFile) reader = csv.reader(open(csvFile)) for row in reader: if reader.line_num == 1: # If we are on the first line, create the headers list from the first row headers = row else: # Otherwise, the key in the content dictionary is the first item in the # row and we can create the sub-dictionary by using the zip() function. # we include the key in the dictionary as our exported csv files use # bookid as the key content[row[0]] = dict(zip(headers, row)) # We can now get to the content by using the resulting dictionary, so to see # the list of lines, we can do: print content.keys() to get a list of keys # To see the list of fields available for each book: print headers if 'Author' not in headers or 'Title' not in headers: msg = 'Invalid CSV file found %s' % csvFile logger.warn(msg) return msg myDB = database.DBConnection() bookcount = 0 authcount = 0 skipcount = 0 logger.debug(u"CSV: Found %s book%s in csv file" % (len(content.keys()), plural(len(content.keys())))) for item in content.keys(): authorname = content[item]['Author'] if isinstance(authorname, str) and hasattr(authorname, "decode"): authorname = authorname.decode(lazylibrarian.SYS_ENCODING) authorname = formatAuthorName(authorname) title = content[item]['Title'] if isinstance(title, str) and hasattr(title, "decode"): title = title.decode(lazylibrarian.SYS_ENCODING) authmatch = myDB.match('SELECT * FROM authors where AuthorName=?', (authorname,)) if authmatch: logger.debug(u"CSV: Author %s found in database" % authorname) else: logger.debug(u"CSV: Author %s not found" % authorname) newauthor, authorid, new = addAuthorNameToDB(author=authorname, addbooks=lazylibrarian.CONFIG['NEWAUTHOR_BOOKS']) if len(newauthor) and newauthor != authorname: logger.debug("Preferred authorname changed from [%s] to [%s]" % (authorname, newauthor)) authorname = newauthor if new: authcount += 1 bookmatch = finditem(content[item], authorname, headers) result = '' if bookmatch: authorname = bookmatch['AuthorName'] bookname = bookmatch['BookName'] bookid = bookmatch['BookID'] bookstatus = bookmatch['Status'] if bookstatus in ['Open', 'Wanted', 'Have']: logger.info(u'Found book %s by %s, already marked as "%s"' % (bookname, authorname, bookstatus)) else: # skipped/ignored logger.info(u'Found book %s by %s, marking as "Wanted"' % (bookname, authorname)) controlValueDict = {"BookID": bookid} newValueDict = {"Status": "Wanted"} myDB.upsert("books", newValueDict, controlValueDict) bookcount += 1 else: searchterm = "%s <ll> %s" % (title, authorname) results = search_for(unaccented(searchterm)) if results: result = results[0] if result['author_fuzz'] > lazylibrarian.CONFIG['MATCH_RATIO'] \ and result['book_fuzz'] > lazylibrarian.CONFIG['MATCH_RATIO']: logger.info("Found (%s%% %s%%) %s: %s" % (result['author_fuzz'], result['book_fuzz'], result['authorname'], result['bookname'])) import_book(result['bookid']) bookcount += 1 bookmatch = True if not bookmatch: msg = "Skipping book %s by %s" % (title, authorname) if not result: msg += ', No results returned' logger.warn(msg) else: msg += ', No match found' logger.warn(msg) msg = "Closest match (%s%% %s%%) %s: %s" % (result['author_fuzz'], result['book_fuzz'], result['authorname'], result['bookname']) logger.warn(msg) skipcount += 1 msg = "Added %i new author%s, marked %i book%s as 'Wanted', %i book%s not found" % \ (authcount, plural(authcount), bookcount, plural(bookcount), skipcount, plural(skipcount)) logger.info(msg) return msg except Exception: msg = 'Unhandled exception in importCSV: %s' % traceback.format_exc() logger.error(msg) return msg
def import_CSV(search_dir=None, library='eBook'): """ Find a csv file in the search_dir and process all the books in it, adding authors to the database if not found and marking the books as "Wanted" Optionally delete the file on successful completion """ # noinspection PyBroadException try: if not search_dir: msg = "Alternate Directory not configured" logger.warn(msg) return msg elif not os.path.isdir(search_dir): msg = "Alternate Directory [%s] not found" % search_dir logger.warn(msg) return msg csvFile = csv_file(search_dir, library=library) headers = None myDB = database.DBConnection() bookcount = 0 authcount = 0 skipcount = 0 total = 0 existing = 0 if not csvFile: msg = "No %s CSV file found in %s" % (library, search_dir) logger.warn(msg) return msg else: logger.debug('Reading file %s' % csvFile) csvreader = reader(open(csvFile, 'rU')) for row in csvreader: if csvreader.line_num == 1: # If we are on the first line, create the headers list from the first row headers = row if 'Author' not in headers or 'Title' not in headers: msg = 'Invalid CSV file found %s' % csvFile logger.warn(msg) return msg else: total += 1 item = dict(list(zip(headers, row))) authorname = formatAuthorName(item['Author']) title = makeUnicode(item['Title']) authmatch = myDB.match('SELECT * FROM authors where AuthorName=?', (authorname,)) if authmatch: logger.debug("CSV: Author %s found in database" % authorname) else: logger.debug("CSV: Author %s not found" % authorname) newauthor, authorid, new = addAuthorNameToDB(author=authorname, addbooks=lazylibrarian.CONFIG['NEWAUTHOR_BOOKS']) if len(newauthor) and newauthor != authorname: logger.debug("Preferred authorname changed from [%s] to [%s]" % (authorname, newauthor)) authorname = newauthor if new: authcount += 1 bookmatch = finditem(item, authorname, library=library) result = '' imported = '' if bookmatch: authorname = bookmatch['AuthorName'] bookname = bookmatch['BookName'] bookid = bookmatch['BookID'] if library == 'eBook': bookstatus = bookmatch['Status'] else: bookstatus = bookmatch['AudioStatus'] if bookstatus in ['Open', 'Wanted', 'Have']: existing += 1 logger.info('Found %s %s by %s, already marked as "%s"' % (library, bookname, authorname, bookstatus)) else: # skipped/ignored logger.info('Found %s %s by %s, marking as "Wanted"' % (library, bookname, authorname)) controlValueDict = {"BookID": bookid} if library == 'eBook': newValueDict = {"Status": "Wanted"} else: newValueDict = {"AudioStatus": "Wanted"} myDB.upsert("books", newValueDict, controlValueDict) bookcount += 1 else: searchterm = "%s <ll> %s" % (title, authorname) results = search_for(unaccented(searchterm)) if results: result = results[0] if result['author_fuzz'] >= lazylibrarian.CONFIG['MATCH_RATIO'] \ and result['book_fuzz'] >= lazylibrarian.CONFIG['MATCH_RATIO']: bookmatch = True if not bookmatch: # no match on full searchterm, try splitting out subtitle newtitle, _ = split_title(authorname, title) if newtitle != title: title = newtitle searchterm = "%s <ll> %s" % (title, authorname) results = search_for(unaccented(searchterm)) if results: result = results[0] if result['author_fuzz'] >= lazylibrarian.CONFIG['MATCH_RATIO'] \ and result['book_fuzz'] >= lazylibrarian.CONFIG['MATCH_RATIO']: bookmatch = True if bookmatch: logger.info("Found (%s%% %s%%) %s: %s for %s: %s" % (result['author_fuzz'], result['book_fuzz'], result['authorname'], result['bookname'], authorname, title)) if library == 'eBook': import_book(result['bookid'], ebook="Wanted", wait=True) else: import_book(result['bookid'], audio="Wanted", wait=True) imported = myDB.match('select * from books where BookID=?', (result['bookid'],)) if imported: bookcount += 1 else: bookmatch = False if not bookmatch: msg = "Skipping book %s by %s" % (title, authorname) if not result: msg += ', No results found' logger.warn(msg) elif not imported: msg += ', Failed to import %s' % result['bookid'] logger.warn(msg) else: msg += ', No match found' logger.warn(msg) msg = "Closest match (%s%% %s%%) %s: %s" % (result['author_fuzz'], result['book_fuzz'], result['authorname'], result['bookname']) logger.warn(msg) skipcount += 1 msg = "Found %i %s%s in csv file, %i already existing or wanted" % (total, library, plural(total), existing) logger.info(msg) msg = "Added %i new author%s, marked %i %s%s as 'Wanted', %i %s%s not found" % \ (authcount, plural(authcount), bookcount, library, plural(bookcount), skipcount, plural(skipcount), library) logger.info(msg) if lazylibrarian.CONFIG['DELETE_CSV']: if skipcount == 0: logger.info("Deleting %s on successful completion" % csvFile) try: os.remove(csvFile) except OSError as why: logger.warn('Unable to delete %s: %s' % (csvFile, why.strerror)) else: logger.warn("Not deleting %s as not all books found" % csvFile) if os.path.isdir(csvFile + '.fail'): try: shutil.rmtree(csvFile + '.fail') except Exception as why: logger.warn("Unable to remove %s, %s %s" % (csvFile + '.fail', type(why).__name__, str(why))) try: _ = safe_move(csvFile, csvFile + '.fail') except Exception as e: logger.error("Unable to rename %s, %s %s" % (csvFile, type(e).__name__, str(e))) if not os.access(csvFile, os.R_OK): logger.error("%s is not readable" % csvFile) if not os.access(csvFile, os.W_OK): logger.error("%s is not writeable" % csvFile) parent = os.path.dirname(csvFile) try: with open(os.path.join(parent, 'll_temp'), 'w') as f: f.write('test') os.remove(os.path.join(parent, 'll_temp')) except Exception as why: logger.error("Directory %s is not writeable: %s" % (parent, why)) return msg except Exception: msg = 'Unhandled exception in importCSV: %s' % traceback.format_exc() logger.error(msg) return msg
def syncCalibreList(col_read=None, col_toread=None, userid=None): """ Get the lazylibrarian bookid for each read/toread calibre book so we can map our id to theirs, and sync current/supplied user's read/toread or supplied read/toread columns to calibre database. Return message giving totals """ myDB = database.DBConnection() if not userid: cookie = cherrypy.request.cookie if cookie and 'll_uid' in cookie.keys(): userid = cookie['ll_uid'].value if userid: res = myDB.match( 'SELECT UserName,ToRead,HaveRead,CalibreRead,CalibreToRead,Perms from users where UserID=?', (userid, )) if res: username = res['UserName'] if not col_read: col_read = res['CalibreRead'] if not col_toread: col_toread = res['CalibreToRead'] toreadlist = getList(res['ToRead']) readlist = getList(res['HaveRead']) # suppress duplicates (just in case) toreadlist = list(set(toreadlist)) readlist = list(set(readlist)) else: return "Error: Unable to get user column settings for %s" % userid if not userid: return "Error: Unable to find current userid" if not col_read and not col_toread: return "User %s has no calibre columns set" % username # check user columns exist in calibre and create if not res = calibredb('custom_columns') columns = res[0].split('\n') custom_columns = [] for column in columns: if column: custom_columns.append(column.split(' (')[0]) if col_read not in custom_columns: added = calibredb('add_custom_column', [col_read, col_read, 'bool']) if "column created" not in added[0]: return added if col_toread not in custom_columns: added = calibredb('add_custom_column', [col_toread, col_toread, 'bool']) if "column created" not in added[0]: return added nomatch = 0 readcol = '' toreadcol = '' map_ctol = {} map_ltoc = {} if col_read: readcol = '*' + col_read if col_toread: toreadcol = '*' + col_toread calibre_list = calibreList(col_read, col_toread) if not isinstance(calibre_list, list): # got an error message from calibredb return '"%s"' % calibre_list for item in calibre_list: if toreadcol and toreadcol in item or readcol and readcol in item: authorname, authorid, added = addAuthorNameToDB(item['authors'], refresh=False, addbooks=False) if authorname: if authorname != item['authors']: logger.debug( "Changed authorname for [%s] from [%s] to [%s]" % (item['title'], item['authors'], authorname)) item['authors'] = authorname bookid = find_book_in_db(authorname, item['title']) if not bookid: searchterm = "%s <ll> %s" % (item['title'], authorname) results = search_for(unaccented(searchterm)) if results: result = results[0] if result['author_fuzz'] > lazylibrarian.CONFIG['MATCH_RATIO'] \ and result['book_fuzz'] > lazylibrarian.CONFIG['MATCH_RATIO']: logger.debug( "Found (%s%% %s%%) %s: %s" % (result['author_fuzz'], result['book_fuzz'], result['authorname'], result['bookname'])) bookid = result['bookid'] import_book(bookid) if bookid: # NOTE: calibre bookid is always an integer, lazylibrarian bookid is a string # (goodreads could be used as an int, but googlebooks can't as it's alphanumeric) # so convert all dict items to strings for ease of matching. map_ctol[str(item['id'])] = str(bookid) map_ltoc[str(bookid)] = str(item['id']) else: logger.warn( 'Calibre Book [%s] by [%s] is not in lazylibrarian database' % (item['title'], authorname)) nomatch += 1 else: logger.warn( 'Calibre Author [%s] not matched in lazylibrarian database' % (item['authors'])) nomatch += 1 # Now check current users lazylibrarian read/toread against the calibre library, warn about missing ones # which might be books calibre doesn't have, or might be minor differences in author or title for idlist in [("Read", readlist), ("To_Read", toreadlist)]: booklist = idlist[1] for bookid in booklist: cmd = "SELECT AuthorID,BookName from books where BookID=?" book = myDB.match(cmd, (bookid, )) if not book: logger.error('Error finding bookid %s' % bookid) else: cmd = "SELECT AuthorName from authors where AuthorID=?" author = myDB.match(cmd, (book['AuthorID'], )) if not author: logger.error('Error finding authorid %s' % book['AuthorID']) else: match = False for item in calibre_list: if item['authors'] == author['AuthorName'] and item[ 'title'] == book['BookName']: logger.debug("Exact match for %s [%s]" % (idlist[0], book['BookName'])) map_ctol[str(item['id'])] = str(bookid) map_ltoc[str(bookid)] = str(item['id']) match = True break if not match: high = 0 highname = '' highid = '' for item in calibre_list: if item['authors'] == author['AuthorName']: n = fuzz.token_sort_ratio( item['title'], book['BookName']) if n > high: high = n highname = item['title'] highid = item['id'] if high > 95: logger.debug( "Found ratio match %s%% [%s] for %s [%s]" % (high, highname, idlist[0], book['BookName'])) map_ctol[str(highid)] = str(bookid) map_ltoc[str(bookid)] = str(highid) match = True if not match: logger.warn( "No match for %s %s by %s in calibre database, closest match %s%% [%s]" % (idlist[0], book['BookName'], author['AuthorName'], high, highname)) nomatch += 1 logger.debug("BookID mapping complete, %s match %s, nomatch %s" % (username, len(map_ctol), nomatch)) # now sync the lists if userid: last_read = [] last_toread = [] calibre_read = [] calibre_toread = [] cmd = 'select SyncList from sync where UserID=? and Label=?' res = myDB.match(cmd, (userid, col_read)) if res: last_read = getList(res['SyncList']) res = myDB.match(cmd, (userid, col_toread)) if res: last_toread = getList(res['SyncList']) for item in calibre_list: if toreadcol and toreadcol in item and item[ toreadcol]: # only if True if str(item['id']) in map_ctol: calibre_toread.append(map_ctol[str(item['id'])]) else: logger.warn( "Calibre to_read book %s:%s has no lazylibrarian bookid" % (item['authors'], item['title'])) if readcol and readcol in item and item[readcol]: # only if True if str(item['id']) in map_ctol: calibre_read.append(map_ctol[str(item['id'])]) else: logger.warn( "Calibre read book %s:%s has no lazylibrarian bookid" % (item['authors'], item['title'])) logger.debug("Found %s calibre read, %s calibre toread" % (len(calibre_read), len(calibre_toread))) logger.debug("Found %s lazylib read, %s lazylib toread" % (len(readlist), len(toreadlist))) added_to_ll_toread = list(set(toreadlist) - set(last_toread)) removed_from_ll_toread = list(set(last_toread) - set(toreadlist)) added_to_ll_read = list(set(readlist) - set(last_read)) removed_from_ll_read = list(set(last_read) - set(readlist)) logger.debug("lazylibrarian changes to copy to calibre: %s %s %s %s" % (len(added_to_ll_toread), len(removed_from_ll_toread), len(added_to_ll_read), len(removed_from_ll_read))) added_to_calibre_toread = list(set(calibre_toread) - set(last_toread)) removed_from_calibre_toread = list( set(last_toread) - set(calibre_toread)) added_to_calibre_read = list(set(calibre_read) - set(last_read)) removed_from_calibre_read = list(set(last_read) - set(calibre_read)) logger.debug( "calibre changes to copy to lazylibrarian: %s %s %s %s" % (len(added_to_calibre_toread), len(removed_from_calibre_toread), len(added_to_calibre_read), len(removed_from_calibre_read))) calibre_changes = 0 for item in added_to_calibre_read: if item not in readlist: readlist.append(item) logger.debug("Lazylibrarian marked %s as read" % item) calibre_changes += 1 for item in added_to_calibre_toread: if item not in toreadlist: toreadlist.append(item) logger.debug("Lazylibrarian marked %s as to_read" % item) calibre_changes += 1 for item in removed_from_calibre_read: if item in readlist: readlist.remove(item) logger.debug("Lazylibrarian removed %s from read" % item) calibre_changes += 1 for item in removed_from_calibre_toread: if item in toreadlist: toreadlist.remove(item) logger.debug("Lazylibrarian removed %s from to_read" % item) calibre_changes += 1 if calibre_changes: myDB.action('UPDATE users SET ToRead=?,HaveRead=? WHERE UserID=?', (', '.join(toreadlist), ', '.join(readlist), userid)) ll_changes = 0 for item in added_to_ll_toread: if item in map_ltoc: res, err, rc = calibredb('set_custom', [col_toread, map_ltoc[item], 'true'], []) if rc: msg = "calibredb set_custom error: " if err: logger.error(msg + err) elif res: logger.error(msg + res) else: logger.error(msg + str(rc)) else: ll_changes += 1 else: logger.warn("Unable to set calibre %s true for %s" % (col_toread, item)) for item in removed_from_ll_toread: if item in map_ltoc: res, err, rc = calibredb('set_custom', [col_toread, map_ltoc[item], ''], []) if rc: msg = "calibredb set_custom error: " if err: logger.error(msg + err) elif res: logger.error(msg + res) else: logger.error(msg + str(rc)) else: ll_changes += 1 else: logger.warn("Unable to clear calibre %s for %s" % (col_toread, item)) for item in added_to_ll_read: if item in map_ltoc: res, err, rc = calibredb('set_custom', [col_read, map_ltoc[item], 'true'], []) if rc: msg = "calibredb set_custom error: " if err: logger.error(msg + err) elif res: logger.error(msg + res) else: logger.error(msg + str(rc)) else: ll_changes += 1 else: logger.warn("Unable to set calibre %s true for %s" % (col_read, item)) for item in removed_from_ll_read: if item in map_ltoc: res, err, rc = calibredb('set_custom', [col_read, map_ltoc[item], ''], []) if rc: msg = "calibredb set_custom error: " if err: logger.error(msg + err) elif res: logger.error(msg + res) else: logger.error(msg + str(rc)) else: ll_changes += 1 else: logger.warn("Unable to clear calibre %s for %s" % (col_read, item)) # store current sync list as comparison for next sync controlValueDict = {"UserID": userid, "Label": col_read} newValueDict = { "Date": str(time.time()), "Synclist": ', '.join(readlist) } myDB.upsert("sync", newValueDict, controlValueDict) controlValueDict = {"UserID": userid, "Label": col_toread} newValueDict = { "Date": str(time.time()), "Synclist": ', '.join(toreadlist) } myDB.upsert("sync", newValueDict, controlValueDict) msg = "%s sync updated: %s calibre, %s lazylibrarian" % ( username, ll_changes, calibre_changes) return msg
def import_CSV(search_dir=None): """ Find a csv file in the search_dir and process all the books in it, adding authors to the database if not found and marking the books as "Wanted" Optionally delete the file on successful completion """ # noinspection PyBroadException try: if not search_dir: msg = "Alternate Directory not configured" logger.warn(msg) return msg elif not os.path.isdir(search_dir): msg = "Alternate Directory [%s] not found" % search_dir logger.warn(msg) return msg csvFile = csv_file(search_dir) headers = None myDB = database.DBConnection() bookcount = 0 authcount = 0 skipcount = 0 total = 0 existing = 0 if not csvFile: msg = "No CSV file found in %s" % search_dir logger.warn(msg) return msg else: logger.debug('Reading file %s' % csvFile) csvreader = reader(open(csvFile, 'rU')) for row in csvreader: if csvreader.line_num == 1: # If we are on the first line, create the headers list from the first row headers = row if 'Author' not in headers or 'Title' not in headers: msg = 'Invalid CSV file found %s' % csvFile logger.warn(msg) return msg else: total += 1 item = dict(list(zip(headers, row))) authorname = formatAuthorName(item['Author']) title = makeUnicode(item['Title']) authmatch = myDB.match( 'SELECT * FROM authors where AuthorName=?', (authorname, )) if authmatch: logger.debug("CSV: Author %s found in database" % authorname) else: logger.debug("CSV: Author %s not found" % authorname) newauthor, authorid, new = addAuthorNameToDB( author=authorname, addbooks=lazylibrarian.CONFIG['NEWAUTHOR_BOOKS']) if len(newauthor) and newauthor != authorname: logger.debug( "Preferred authorname changed from [%s] to [%s]" % (authorname, newauthor)) authorname = newauthor if new: authcount += 1 bookmatch = finditem(item, authorname) result = '' imported = '' if bookmatch: authorname = bookmatch['AuthorName'] bookname = bookmatch['BookName'] bookid = bookmatch['BookID'] bookstatus = bookmatch['Status'] if bookstatus in ['Open', 'Wanted', 'Have']: existing += 1 logger.info( 'Found book %s by %s, already marked as "%s"' % (bookname, authorname, bookstatus)) else: # skipped/ignored logger.info( 'Found book %s by %s, marking as "Wanted"' % (bookname, authorname)) controlValueDict = {"BookID": bookid} newValueDict = {"Status": "Wanted"} myDB.upsert("books", newValueDict, controlValueDict) bookcount += 1 else: searchterm = "%s <ll> %s" % (title, authorname) results = search_for(unaccented(searchterm)) if results: result = results[0] if result['author_fuzz'] >= lazylibrarian.CONFIG['MATCH_RATIO'] \ and result['book_fuzz'] >= lazylibrarian.CONFIG['MATCH_RATIO']: bookmatch = True if not bookmatch: # no match on full searchterm, try splitting out subtitle newtitle, _ = split_title(authorname, title) if newtitle != title: title = newtitle searchterm = "%s <ll> %s" % (title, authorname) results = search_for(unaccented(searchterm)) if results: result = results[0] if result['author_fuzz'] >= lazylibrarian.CONFIG['MATCH_RATIO'] \ and result['book_fuzz'] >= lazylibrarian.CONFIG['MATCH_RATIO']: bookmatch = True if bookmatch: logger.info( "Found (%s%% %s%%) %s: %s for %s: %s" % (result['author_fuzz'], result['book_fuzz'], result['authorname'], result['bookname'], authorname, title)) import_book(result['bookid'], wait=True) imported = myDB.match( 'select * from books where BookID=?', (result['bookid'], )) if imported: bookcount += 1 else: bookmatch = False if not bookmatch: msg = "Skipping book %s by %s" % (title, authorname) if not result: msg += ', No results found' logger.warn(msg) elif not imported: msg += ', Failed to import %s' % result['bookid'] logger.warn(msg) else: msg += ', No match found' logger.warn(msg) msg = "Closest match (%s%% %s%%) %s: %s" % ( result['author_fuzz'], result['book_fuzz'], result['authorname'], result['bookname']) logger.warn(msg) skipcount += 1 msg = "Found %i book%s in csv file, %i already existing or wanted" % ( total, plural(total), existing) logger.info(msg) msg = "Added %i new author%s, marked %i book%s as 'Wanted', %i book%s not found" % \ (authcount, plural(authcount), bookcount, plural(bookcount), skipcount, plural(skipcount)) logger.info(msg) if lazylibrarian.CONFIG['DELETE_CSV']: if skipcount == 0: logger.info("Deleting %s on successful completion" % csvFile) try: os.remove(csvFile) except OSError as why: logger.warn('Unable to delete %s: %s' % (csvFile, why.strerror)) else: logger.warn("Not deleting %s as not all books found" % csvFile) return msg except Exception: msg = 'Unhandled exception in importCSV: %s' % traceback.format_exc() logger.error(msg) return msg
def syncCalibreList(col_read=None, col_toread=None, userid=None): """ Get the lazylibrarian bookid for each read/toread calibre book so we can map our id to theirs, and sync current/supplied user's read/toread or supplied read/toread columns to calibre database. Return message giving totals """ myDB = database.DBConnection() username = '' readlist = [] toreadlist = [] if not userid: cookie = cherrypy.request.cookie if cookie and 'll_uid' in list(cookie.keys()): userid = cookie['ll_uid'].value if userid: res = myDB.match('SELECT UserName,ToRead,HaveRead,CalibreRead,CalibreToRead,Perms from users where UserID=?', (userid,)) if res: username = res['UserName'] if not col_read: col_read = res['CalibreRead'] if not col_toread: col_toread = res['CalibreToRead'] toreadlist = getList(res['ToRead']) readlist = getList(res['HaveRead']) # suppress duplicates (just in case) toreadlist = list(set(toreadlist)) readlist = list(set(readlist)) else: return "Error: Unable to get user column settings for %s" % userid if not userid: return "Error: Unable to find current userid" if not col_read and not col_toread: return "User %s has no calibre columns set" % username # check user columns exist in calibre and create if not res = calibredb('custom_columns') columns = res[0].split('\n') custom_columns = [] for column in columns: if column: custom_columns.append(column.split(' (')[0]) if col_read not in custom_columns: added = calibredb('add_custom_column', [col_read, col_read, 'bool']) if "column created" not in added[0]: return added if col_toread not in custom_columns: added = calibredb('add_custom_column', [col_toread, col_toread, 'bool']) if "column created" not in added[0]: return added nomatch = 0 readcol = '' toreadcol = '' map_ctol = {} map_ltoc = {} if col_read: readcol = '*' + col_read if col_toread: toreadcol = '*' + col_toread calibre_list = calibreList(col_read, col_toread) if not isinstance(calibre_list, list): # got an error message from calibredb return '"%s"' % calibre_list for item in calibre_list: if toreadcol and toreadcol in item or readcol and readcol in item: authorname, authorid, added = addAuthorNameToDB(item['authors'], refresh=False, addbooks=False) if authorname: if authorname != item['authors']: logger.debug("Changed authorname for [%s] from [%s] to [%s]" % (item['title'], item['authors'], authorname)) item['authors'] = authorname bookid, mtype = find_book_in_db(authorname, item['title'], ignored=False, library='eBook') if bookid and mtype == "Ignored": logger.warn("Book %s by %s is marked Ignored in database, importing anyway" % (item['title'], authorname)) if not bookid: searchterm = "%s <ll> %s" % (item['title'], authorname) results = search_for(unaccented(searchterm)) if results: result = results[0] if result['author_fuzz'] > lazylibrarian.CONFIG['MATCH_RATIO'] \ and result['book_fuzz'] > lazylibrarian.CONFIG['MATCH_RATIO']: logger.debug("Found (%s%% %s%%) %s: %s" % (result['author_fuzz'], result['book_fuzz'], result['authorname'], result['bookname'])) bookid = result['bookid'] import_book(bookid) if bookid: # NOTE: calibre bookid is always an integer, lazylibrarian bookid is a string # (goodreads could be used as an int, but googlebooks can't as it's alphanumeric) # so convert all dict items to strings for ease of matching. map_ctol[str(item['id'])] = str(bookid) map_ltoc[str(bookid)] = str(item['id']) else: logger.warn('Calibre Book [%s] by [%s] is not in lazylibrarian database' % (item['title'], authorname)) nomatch += 1 else: logger.warn('Calibre Author [%s] not matched in lazylibrarian database' % (item['authors'])) nomatch += 1 # Now check current users lazylibrarian read/toread against the calibre library, warn about missing ones # which might be books calibre doesn't have, or might be minor differences in author or title for idlist in [("Read", readlist), ("To_Read", toreadlist)]: booklist = idlist[1] for bookid in booklist: cmd = "SELECT AuthorID,BookName from books where BookID=?" book = myDB.match(cmd, (bookid,)) if not book: logger.error('Error finding bookid %s' % bookid) else: cmd = "SELECT AuthorName from authors where AuthorID=?" author = myDB.match(cmd, (book['AuthorID'],)) if not author: logger.error('Error finding authorid %s' % book['AuthorID']) else: match = False high = 0 highname = '' for item in calibre_list: if item['authors'] == author['AuthorName'] and item['title'] == book['BookName']: logger.debug("Exact match for %s [%s]" % (idlist[0], book['BookName'])) map_ctol[str(item['id'])] = str(bookid) map_ltoc[str(bookid)] = str(item['id']) match = True break if not match: highid = '' for item in calibre_list: if item['authors'] == author['AuthorName']: n = fuzz.token_sort_ratio(item['title'], book['BookName']) if n > high: high = n highname = item['title'] highid = item['id'] if high > 95: logger.debug("Found ratio match %s%% [%s] for %s [%s]" % (high, highname, idlist[0], book['BookName'])) map_ctol[str(highid)] = str(bookid) map_ltoc[str(bookid)] = str(highid) match = True if not match: logger.warn("No match for %s %s by %s in calibre database, closest match %s%% [%s]" % (idlist[0], book['BookName'], author['AuthorName'], high, highname)) nomatch += 1 logger.debug("BookID mapping complete, %s match %s, nomatch %s" % (username, len(map_ctol), nomatch)) # now sync the lists if not userid: msg = "No userid found" else: last_read = [] last_toread = [] calibre_read = [] calibre_toread = [] cmd = 'select SyncList from sync where UserID=? and Label=?' res = myDB.match(cmd, (userid, col_read)) if res: last_read = getList(res['SyncList']) res = myDB.match(cmd, (userid, col_toread)) if res: last_toread = getList(res['SyncList']) for item in calibre_list: if toreadcol and toreadcol in item and item[toreadcol]: # only if True if str(item['id']) in map_ctol: calibre_toread.append(map_ctol[str(item['id'])]) else: logger.warn("Calibre to_read book %s:%s has no lazylibrarian bookid" % (item['authors'], item['title'])) if readcol and readcol in item and item[readcol]: # only if True if str(item['id']) in map_ctol: calibre_read.append(map_ctol[str(item['id'])]) else: logger.warn("Calibre read book %s:%s has no lazylibrarian bookid" % (item['authors'], item['title'])) logger.debug("Found %s calibre read, %s calibre toread" % (len(calibre_read), len(calibre_toread))) logger.debug("Found %s lazylib read, %s lazylib toread" % (len(readlist), len(toreadlist))) added_to_ll_toread = list(set(toreadlist) - set(last_toread)) removed_from_ll_toread = list(set(last_toread) - set(toreadlist)) added_to_ll_read = list(set(readlist) - set(last_read)) removed_from_ll_read = list(set(last_read) - set(readlist)) logger.debug("lazylibrarian changes to copy to calibre: %s %s %s %s" % (len(added_to_ll_toread), len(removed_from_ll_toread), len(added_to_ll_read), len(removed_from_ll_read))) added_to_calibre_toread = list(set(calibre_toread) - set(last_toread)) removed_from_calibre_toread = list(set(last_toread) - set(calibre_toread)) added_to_calibre_read = list(set(calibre_read) - set(last_read)) removed_from_calibre_read = list(set(last_read) - set(calibre_read)) logger.debug("calibre changes to copy to lazylibrarian: %s %s %s %s" % (len(added_to_calibre_toread), len(removed_from_calibre_toread), len(added_to_calibre_read), len(removed_from_calibre_read))) calibre_changes = 0 for item in added_to_calibre_read: if item not in readlist: readlist.append(item) logger.debug("Lazylibrarian marked %s as read" % item) calibre_changes += 1 for item in added_to_calibre_toread: if item not in toreadlist: toreadlist.append(item) logger.debug("Lazylibrarian marked %s as to_read" % item) calibre_changes += 1 for item in removed_from_calibre_read: if item in readlist: readlist.remove(item) logger.debug("Lazylibrarian removed %s from read" % item) calibre_changes += 1 for item in removed_from_calibre_toread: if item in toreadlist: toreadlist.remove(item) logger.debug("Lazylibrarian removed %s from to_read" % item) calibre_changes += 1 if calibre_changes: myDB.action('UPDATE users SET ToRead=?,HaveRead=? WHERE UserID=?', (', '.join(toreadlist), ', '.join(readlist), userid)) ll_changes = 0 for item in added_to_ll_toread: if item in map_ltoc: res, err, rc = calibredb('set_custom', [col_toread, map_ltoc[item], 'true'], []) if rc: msg = "calibredb set_custom error: " if err: logger.error(msg + err) elif res: logger.error(msg + res) else: logger.error(msg + str(rc)) else: ll_changes += 1 else: logger.warn("Unable to set calibre %s true for %s" % (col_toread, item)) for item in removed_from_ll_toread: if item in map_ltoc: res, err, rc = calibredb('set_custom', [col_toread, map_ltoc[item], ''], []) if rc: msg = "calibredb set_custom error: " if err: logger.error(msg + err) elif res: logger.error(msg + res) else: logger.error(msg + str(rc)) else: ll_changes += 1 else: logger.warn("Unable to clear calibre %s for %s" % (col_toread, item)) for item in added_to_ll_read: if item in map_ltoc: res, err, rc = calibredb('set_custom', [col_read, map_ltoc[item], 'true'], []) if rc: msg = "calibredb set_custom error: " if err: logger.error(msg + err) elif res: logger.error(msg + res) else: logger.error(msg + str(rc)) else: ll_changes += 1 else: logger.warn("Unable to set calibre %s true for %s" % (col_read, item)) for item in removed_from_ll_read: if item in map_ltoc: res, err, rc = calibredb('set_custom', [col_read, map_ltoc[item], ''], []) if rc: msg = "calibredb set_custom error: " if err: logger.error(msg + err) elif res: logger.error(msg + res) else: logger.error(msg + str(rc)) else: ll_changes += 1 else: logger.warn("Unable to clear calibre %s for %s" % (col_read, item)) # store current sync list as comparison for next sync controlValueDict = {"UserID": userid, "Label": col_read} newValueDict = {"Date": str(time.time()), "Synclist": ', '.join(readlist)} myDB.upsert("sync", newValueDict, controlValueDict) controlValueDict = {"UserID": userid, "Label": col_toread} newValueDict = {"Date": str(time.time()), "Synclist": ', '.join(toreadlist)} myDB.upsert("sync", newValueDict, controlValueDict) msg = "%s sync updated: %s calibre, %s lazylibrarian" % (username, ll_changes, calibre_changes) return msg
def LibraryScan(startdir=None): """ Scan a directory tree adding new books into database Return how many books you added """ try: destdir = lazylibrarian.DIRECTORY('Destination') if not startdir: if not destdir: logger.warn('Cannot find destination directory: %s. Not scanning' % destdir) return 0 startdir = destdir if not os.path.isdir(startdir): logger.warn('Cannot find directory: %s. Not scanning' % startdir) return 0 if not internet(): logger.warn('Libraryscan: No internet connection') return 0 myDB = database.DBConnection() # keep statistics of full library scans if startdir == destdir: myDB.action('DELETE from stats') try: # remove any extra whitespace in authornames authors = myDB.select('SELECT AuthorID,AuthorName FROM authors WHERE AuthorName like "% %"') if authors: logger.info('Removing extra spaces from %s authorname%s' % (len(authors), plural(len(authors)))) for author in authors: authorid = author["AuthorID"] authorname = ' '.join(author['AuthorName'].split()) # Have we got author name both with-and-without extra spaces? If so, merge them duplicate = myDB.match( 'Select AuthorID,AuthorName FROM authors WHERE AuthorName="%s"' % authorname) if duplicate: myDB.action('DELETE from authors where authorname="%s"' % author['AuthorName']) if author['AuthorID'] != duplicate['AuthorID']: myDB.action('UPDATE books set AuthorID="%s" WHERE AuthorID="%s"' % (duplicate['AuthorID'], author['AuthorID'])) else: myDB.action( 'UPDATE authors set AuthorName="%s" WHERE AuthorID="%s"' % (authorname, authorid)) except Exception as e: logger.info('Error: ' + str(e)) logger.info('Scanning ebook directory: %s' % startdir) new_book_count = 0 modified_count = 0 rescan_count = 0 rescan_hits = 0 file_count = 0 author = "" if lazylibrarian.CONFIG['FULL_SCAN']: cmd = 'select AuthorName, BookName, BookFile, BookID from books,authors' cmd += ' where books.AuthorID = authors.AuthorID and books.Status="Open"' if not startdir == destdir: cmd += ' and BookFile like "' + startdir + '%"' books = myDB.select(cmd) status = lazylibrarian.CONFIG['NOTFOUND_STATUS'] logger.info('Missing books will be marked as %s' % status) for book in books: bookID = book['BookID'] bookfile = book['BookFile'] if not (bookfile and os.path.isfile(bookfile)): myDB.action('update books set Status="%s" where BookID="%s"' % (status, bookID)) myDB.action('update books set BookFile="" where BookID="%s"' % bookID) logger.warn('Book %s - %s updated as not found on disk' % (book['AuthorName'], book['BookName'])) # to save repeat-scans of the same directory if it contains multiple formats of the same book, # keep track of which directories we've already looked at processed_subdirectories = [] warned = False # have we warned about no new authors setting matchString = '' for char in lazylibrarian.CONFIG['EBOOK_DEST_FILE']: matchString = matchString + '\\' + char # massage the EBOOK_DEST_FILE config parameter into something we can use # with regular expression matching booktypes = '' count = -1 booktype_list = getList(lazylibrarian.CONFIG['EBOOK_TYPE']) for book_type in booktype_list: count += 1 if count == 0: booktypes = book_type else: booktypes = booktypes + '|' + book_type matchString = matchString.replace("\\$\\A\\u\\t\\h\\o\\r", "(?P<author>.*?)").replace( "\\$\\T\\i\\t\\l\\e", "(?P<book>.*?)") + '\.[' + booktypes + ']' pattern = re.compile(matchString, re.VERBOSE) for r, d, f in os.walk(startdir): for directory in d[:]: # prevent magazine being scanned if directory.startswith("_") or directory.startswith("."): d.remove(directory) for files in f: file_count += 1 if isinstance(r, str): r = r.decode(lazylibrarian.SYS_ENCODING) subdirectory = r.replace(startdir, '') # Added new code to skip if we've done this directory before. # Made this conditional with a switch in config.ini # in case user keeps multiple different books in the same subdirectory if lazylibrarian.CONFIG['IMP_SINGLEBOOK'] and (subdirectory in processed_subdirectories): logger.debug("[%s] already scanned" % subdirectory) else: # If this is a book, try to get author/title/isbn/language # if epub or mobi, read metadata from the book # If metadata.opf exists, use that allowing it to override # embedded metadata. User may have edited metadata.opf # to merge author aliases together # If all else fails, try pattern match for author/title # and look up isbn/lang from LT or GR later match = 0 if is_valid_booktype(files): logger.debug("[%s] Now scanning subdirectory %s" % (startdir, subdirectory)) language = "Unknown" isbn = "" book = "" author = "" gr_id = "" gb_id = "" extn = os.path.splitext(files)[1] # if it's an epub or a mobi we can try to read metadata from it if (extn == ".epub") or (extn == ".mobi"): book_filename = os.path.join(r, files).encode(lazylibrarian.SYS_ENCODING) try: res = get_book_info(book_filename) except Exception as e: logger.debug('get_book_info failed for %s, %s' % (book_filename, str(e))) res = {} # title and creator are the minimum we need if 'title' in res and 'creator' in res: book = res['title'] author = res['creator'] if book and len(book) > 2 and author and len(author) > 2: match = 1 if 'language' in res: language = res['language'] if 'identifier' in res: isbn = res['identifier'] if 'type' in res: extn = res['type'] logger.debug("book meta [%s] [%s] [%s] [%s] [%s]" % (isbn, language, author, book, extn)) if not match: logger.debug("Book meta incomplete in %s" % book_filename) # calibre uses "metadata.opf", LL uses "bookname - authorname.opf" # just look for any .opf file in the current directory since we don't know # LL preferred authorname/bookname at this point. # Allow metadata in file to override book contents as may be users pref metafile = opf_file(r) try: res = get_book_info(metafile) except Exception as e: logger.debug('get_book_info failed for %s, %s' % (metafile, str(e))) res = {} # title and creator are the minimum we need if 'title' in res and 'creator' in res: book = res['title'] author = res['creator'] if book and len(book) > 2 and author and len(author) > 2: match = 1 if 'language' in res: language = res['language'] if 'identifier' in res: isbn = res['identifier'] if 'gr_id' in res: gr_id = res['gr_id'] logger.debug("file meta [%s] [%s] [%s] [%s] [%s]" % (isbn, language, author, book, gr_id)) if not match: logger.debug("File meta incomplete in %s" % metafile) if not match: # no author/book from metadata file, and not embedded either match = pattern.match(files) if match: author = match.group("author") book = match.group("book") if len(book) <= 2 or len(author) <= 2: match = 0 if not match: logger.debug("Pattern match failed [%s]" % files) if match: # flag that we found a book in this subdirectory processed_subdirectories.append(subdirectory) # If we have a valid looking isbn, and language != "Unknown", add it to cache if language != "Unknown" and is_valid_isbn(isbn): logger.debug("Found Language [%s] ISBN [%s]" % (language, isbn)) # we need to add it to language cache if not already # there, is_valid_isbn has checked length is 10 or 13 if len(isbn) == 10: isbnhead = isbn[0:3] else: isbnhead = isbn[3:6] match = myDB.match('SELECT lang FROM languages where isbn = "%s"' % isbnhead) if not match: myDB.action('insert into languages values ("%s", "%s")' % (isbnhead, language)) logger.debug("Cached Lang [%s] ISBN [%s]" % (language, isbnhead)) else: logger.debug("Already cached Lang [%s] ISBN [%s]" % (language, isbnhead)) author, authorid, new = addAuthorNameToDB(author) # get the author name as we know it... if author: # author exists, check if this book by this author is in our database # metadata might have quotes in book name # some books might be stored under a different author name # eg books by multiple authors, books where author is "writing as" # or books we moved to "merge" authors book = book.replace("'", "") # First try and find it under author and bookname # as we may have it under a different bookid or isbn to goodreads/googlebooks # which might have several bookid/isbn for the same book bookid = find_book_in_db(myDB, author, book) if not bookid: # Title or author name might not match or multiple authors # See if the gr_id, gb_id is already in our database if gr_id: bookid = gr_id elif gb_id: bookid = gb_id else: bookid = "" if bookid: match = myDB.match('SELECT BookID FROM books where BookID = "%s"' % bookid) if not match: msg = 'Unable to find book %s by %s in database, trying to add it using ' if bookid == gr_id: msg += "GoodReads ID " + gr_id if bookid == gb_id: msg += "GoogleBooks ID " + gb_id logger.debug(msg % (book, author)) if lazylibrarian.CONFIG['BOOK_API'] == "GoodReads" and gr_id: GR_ID = GoodReads(gr_id) GR_ID.find_book(gr_id, None) elif lazylibrarian.CONFIG['BOOK_API'] == "GoogleBooks" and gb_id: GB_ID = GoogleBooks(gb_id) GB_ID.find_book(gb_id, None) # see if it's there now... match = myDB.match('SELECT BookID from books where BookID="%s"' % bookid) if not match: logger.debug("Unable to add bookid %s to database" % bookid) bookid = "" if not bookid and isbn: # See if the isbn is in our database match = myDB.match('SELECT BookID FROM books where BookIsbn = "%s"' % isbn) if match: bookid = match['BookID'] if not bookid: # get author name from parent directory of this book directory newauthor = os.path.basename(os.path.dirname(r)) # calibre replaces trailing periods with _ eg Smith Jr. -> Smith Jr_ if newauthor.endswith('_'): newauthor = newauthor[:-1] + '.' if author.lower() != newauthor.lower(): logger.debug("Trying authorname [%s]" % newauthor) bookid = find_book_in_db(myDB, newauthor, book) if bookid: logger.warn("%s not found under [%s], found under [%s]" % (book, author, newauthor)) # at this point if we still have no bookid, it looks like we # have author and book title but no database entry for it if not bookid: if lazylibrarian.CONFIG['BOOK_API'] == "GoodReads": # Either goodreads doesn't have the book or it didn't match language prefs # Since we have the book anyway, try and reload it ignoring language prefs rescan_count += 1 base_url = 'http://www.goodreads.com/search.xml?q=' params = {"key": lazylibrarian.CONFIG['GR_API']} if author[1] in '. ': surname = author forename = '' while surname[1] in '. ': forename = forename + surname[0] + '.' surname = surname[2:].strip() if author != forename + ' ' + surname: logger.debug('Stripped authorname [%s] to [%s %s]' % (author, forename, surname)) author = forename + ' ' + surname author = ' '.join(author.split()) # ensure no extra whitespace searchname = author + ' ' + book searchname = cleanName(unaccented(searchname)) searchterm = urllib.quote_plus(searchname.encode(lazylibrarian.SYS_ENCODING)) set_url = base_url + searchterm + '&' + urllib.urlencode(params) try: rootxml, in_cache = get_xml_request(set_url) if not len(rootxml): logger.debug("Error requesting results from GoodReads") else: resultxml = rootxml.getiterator('work') for item in resultxml: booktitle = item.find('./best_book/title').text book_fuzz = fuzz.token_set_ratio(booktitle, book) if book_fuzz >= 98: logger.debug("Rescan found %s : %s" % (booktitle, language)) rescan_hits += 1 bookid = item.find('./best_book/id').text GR_ID = GoodReads(bookid) GR_ID.find_book(bookid, None) if language and language != "Unknown": # set language from book metadata logger.debug("Setting language from metadata %s : %s" % (booktitle, language)) myDB.action('UPDATE books SET BookLang="%s" WHERE BookID="%s"' % (language, bookid)) break if not bookid: logger.warn("GoodReads doesn't know about %s" % book) except Exception as e: logger.error("Error finding rescan results: %s" % str(e)) elif lazylibrarian.CONFIG['BOOK_API'] == "GoogleBooks": # if we get here using googlebooks it's because googlebooks # doesn't have the book. No point in looking for it again. logger.warn("GoogleBooks doesn't know about %s" % book) # see if it's there now... if bookid: cmd = 'SELECT books.Status, BookFile, AuthorName, BookName from books,authors ' cmd += 'where books.AuthorID = authors.AuthorID and BookID="%s"' % bookid check_status = myDB.match(cmd) if not check_status: logger.debug('Unable to find bookid %s in database' % bookid) else: if check_status['Status'] != 'Open': # we found a new book new_book_count += 1 myDB.action( 'UPDATE books set Status="Open" where BookID="%s"' % bookid) # store book location so we can check if it gets removed book_filename = os.path.join(r, files) if not check_status['BookFile']: # no previous location myDB.action('UPDATE books set BookFile="%s" where BookID="%s"' % (book_filename, bookid)) # location may have changed since last scan elif book_filename != check_status['BookFile']: modified_count += 1 logger.warn("Updating book location for %s %s from %s to %s" % (author, book, check_status['BookFile'], book_filename)) logger.debug("%s %s matched %s BookID %s, [%s][%s]" % (author, book, check_status['Status'], bookid, check_status['AuthorName'], check_status['BookName'])) myDB.action('UPDATE books set BookFile="%s" where BookID="%s"' % (book_filename, bookid)) # update cover file to cover.jpg in book folder (if exists) bookdir = os.path.dirname(book_filename) coverimg = os.path.join(bookdir, 'cover.jpg') if os.path.isfile(coverimg): cachedir = lazylibrarian.CACHEDIR cacheimg = os.path.join(cachedir, 'book', bookid + '.jpg') copyfile(coverimg, cacheimg) else: logger.warn( "Failed to match book [%s] by [%s] in database" % (book, author)) else: if not warned and not lazylibrarian.CONFIG['ADD_AUTHOR']: logger.warn("Add authors to database is disabled") warned = True logger.info("%s/%s new/modified book%s found and added to the database" % (new_book_count, modified_count, plural(new_book_count + modified_count))) logger.info("%s file%s processed" % (file_count, plural(file_count))) if startdir == destdir: # On full library scans, check for missing workpages setWorkPages() # and books with unknown language nolang = myDB.match( "select count('BookID') as counter from Books where status='Open' and BookLang='Unknown'") nolang = nolang['counter'] if nolang: logger.warn("Found %s book%s in your library with unknown language" % (nolang, plural(nolang))) # show stats if new books were added stats = myDB.match( "SELECT sum(GR_book_hits), sum(GR_lang_hits), sum(LT_lang_hits), sum(GB_lang_change), \ sum(cache_hits), sum(bad_lang), sum(bad_char), sum(uncached), sum(duplicates) FROM stats") st= {'GR_book_hits': stats['sum(GR_book_hits)'], 'GB_book_hits': stats['sum(GR_book_hits)'], 'GR_lang_hits': stats['sum(GR_lang_hits)'], 'LT_lang_hits': stats['sum(LT_lang_hits)'], 'GB_lang_change': stats['sum(GB_lang_change)'], 'cache_hits': stats['sum(cache_hits)'], 'bad_lang': stats['sum(bad_lang)'], 'bad_char': stats['sum(bad_char)'], 'uncached': stats['sum(uncached)'], 'duplicates': stats['sum(duplicates)']} for item in st.keys(): if st[item] is None: st[item] = 0 if lazylibrarian.CONFIG['BOOK_API'] == "GoogleBooks": logger.debug("GoogleBooks was hit %s time%s for books" % (st['GR_book_hits'], plural(st['GR_book_hits']))) logger.debug("GoogleBooks language was changed %s time%s" % (st['GB_lang_change'], plural(st['GB_lang_change']))) if lazylibrarian.CONFIG['BOOK_API'] == "GoodReads": logger.debug("GoodReads was hit %s time%s for books" % (st['GR_book_hits'], plural(st['GR_book_hits']))) logger.debug("GoodReads was hit %s time%s for languages" % (st['GR_lang_hits'], plural(st['GR_lang_hits']))) logger.debug("LibraryThing was hit %s time%s for languages" % (st['LT_lang_hits'], plural(st['LT_lang_hits']))) logger.debug("Language cache was hit %s time%s" % (st['cache_hits'], plural(st['cache_hits']))) logger.debug("Unwanted language removed %s book%s" % (st['bad_lang'], plural(st['bad_lang']))) logger.debug("Unwanted characters removed %s book%s" % (st['bad_char'], plural(st['bad_char']))) logger.debug("Unable to cache language for %s book%s with missing ISBN" % (st['uncached'], plural(st['uncached']))) logger.debug("Found %s duplicate book%s" % (st['duplicates'], plural(st['duplicates']))) logger.debug("Rescan %s hit%s, %s miss" % (rescan_hits, plural(rescan_hits), rescan_count - rescan_hits)) logger.debug("Cache %s hit%s, %s miss" % (lazylibrarian.CACHE_HIT, plural(lazylibrarian.CACHE_HIT), lazylibrarian.CACHE_MISS)) cachesize = myDB.match("select count('ISBN') as counter from languages") logger.debug("ISBN Language cache holds %s entries" % cachesize['counter']) # Cache any covers and images images = myDB.select('select bookid, bookimg, bookname from books where bookimg like "http%"') if len(images): logger.info("Caching cover%s for %i book%s" % (plural(len(images)), len(images), plural(len(images)))) for item in images: bookid = item['bookid'] bookimg = item['bookimg'] # bookname = item['bookname'] newimg, success = cache_img("book", bookid, bookimg) if success: myDB.action('update books set BookImg="%s" where BookID="%s"' % (newimg, bookid)) images = myDB.select('select AuthorID, AuthorImg, AuthorName from authors where AuthorImg like "http%"') if len(images): logger.info("Caching image%s for %i author%s" % (plural(len(images)), len(images), plural(len(images)))) for item in images: authorid = item['authorid'] authorimg = item['authorimg'] # authorname = item['authorname'] newimg, success = cache_img("author", authorid, authorimg) if success: myDB.action('update authors set AuthorImg="%s" where AuthorID="%s"' % (newimg, authorid)) # On full scan, update bookcounts for all authors, not just new ones - refresh may have located # new books for existing authors especially if switched provider gb/gr or changed wanted languages authors = myDB.select('select AuthorID from authors') else: # On single author/book import, just update bookcount for that author authors = myDB.select('select AuthorID from authors where AuthorName = "%s"' % author.replace('"', '""')) logger.debug('Updating bookcounts for %i author%s' % (len(authors), plural(len(authors)))) for author in authors: update_totals(author['AuthorID']) logger.info('Library scan complete') return new_book_count except Exception: logger.error('Unhandled exception in libraryScan: %s' % traceback.format_exc())