def finditem(item, preferred_authorname): """ Try to find book matching the csv item in the database Return database entry, or False if not found """ myDB = database.DBConnection() bookmatch = "" isbn10 = "" isbn13 = "" bookid = "" bookname = item['Title'] bookname = makeUnicode(bookname) if 'ISBN' in item: isbn10 = item['ISBN'] if 'ISBN13' in item: isbn13 = item['ISBN13'] if 'BookID' in item: bookid = item['BookID'] # try to find book in our database using bookid or isbn, or if that fails, name matching cmd = 'SELECT AuthorName,BookName,BookID,books.Status FROM books,authors where books.AuthorID = authors.AuthorID ' if bookid: fullcmd = cmd + 'and BookID=?' bookmatch = myDB.match(fullcmd, (bookid,)) if not bookmatch: if is_valid_isbn(isbn10): fullcmd = cmd + 'and BookIsbn=?' bookmatch = myDB.match(fullcmd, (isbn10,)) if not bookmatch: if is_valid_isbn(isbn13): fullcmd = cmd + 'and BookIsbn=?' bookmatch = myDB.match(fullcmd, (isbn13,)) if not bookmatch: bookid = find_book_in_db(preferred_authorname, bookname, ignored=False) if not bookid: bookid = find_book_in_db(preferred_authorname, bookname, ignored=True) if bookid: logger.warn("Book %s by %s is marked Ignored in database, importing anyway" % (bookname, preferred_authorname)) if bookid: fullcmd = cmd + 'and BookID=?' bookmatch = myDB.match(fullcmd, (bookid,)) return bookmatch
def processAlternate(source_dir=None): # import a book from an alternate directory if not source_dir or os.path.isdir(source_dir) is False: logger.warn('Alternate directory must not be empty') return if source_dir == lazylibrarian.DESTINATION_DIR: logger.warn('Alternate directory must not be the same as destination') return new_book = book_file(source_dir, booktype='book') if new_book: # see if there is a metadata file in this folder with the info we need metafile = librarysync.opf_file(source_dir) try: metadata = librarysync.get_book_info(metafile) except: metadata = {} if 'title' in metadata and 'creator' in metadata: authorname = metadata['creator'] bookname = metadata['title'] # if not, try to get metadata from the book file else: try: metadata = librarysync.get_book_info(new_book) except: metadata = {} if 'title' in metadata and 'creator' in metadata: authorname = metadata['creator'] bookname = metadata['title'] myDB = database.DBConnection() authmatch = myDB.action( 'SELECT * FROM authors where AuthorName="%s"' % (authorname)).fetchone() if authmatch: logger.debug("ALT: Author %s found in database" % (authorname)) else: logger.debug("ALT: Author %s not found, adding to database" % (authorname)) importer.addAuthorToDB(authorname) bookid = librarysync.find_book_in_db(myDB, authorname, bookname) if bookid: import_book(source_dir, bookid) else: logger.warn("Book %s by %s not found in database" % (bookname, authorname)) else: logger.warn('Book %s has no metadata, unable to import' % new_book) else: logger.warn("No book file found in %s" % source_dir)
def processAlternate(source_dir=None): # import a book from an alternate directory if source_dir == None or source_dir == "": logger.warn('Alternate directory must not be empty') return if source_dir == lazylibrarian.DESTINATION_DIR: logger.warn('Alternate directory must not be the same as destination') return new_book = book_file(source_dir) if new_book: # see if there is a metadata file in this folder with the info we need metafile = librarysync.opf_file(source_dir) try: metadata = librarysync.get_book_info(metafile) except: metadata = {} if 'title' in metadata and 'creator' in metadata: authorname = metadata['creator'] bookname = metadata['title'] # if not, try to get metadata from the book file else: try: metadata = librarysync.get_book_info(new_book) except: metadata = {} if 'title' in metadata and 'creator' in metadata: authorname = metadata['creator'] bookname = metadata['title'] myDB = database.DBConnection() authmatch = myDB.action('SELECT * FROM authors where AuthorName="%s"' % (authorname)).fetchone() if authmatch: logger.debug("ALT: Author %s found in database" % (authorname)) else: logger.debug("ALT: Author %s not found, adding to database" % (authorname)) importer.addAuthorToDB(authorname) bookid = librarysync.find_book_in_db(myDB, authorname, bookname) if bookid: import_book(source_dir, bookid) else: logger.warn("Book %s by %s not found in database" % (bookname, authorname)) else: logger.warn('Book %s has no metadata, unable to import' % new_book) else: logger.warn("No book file found in %s" % source_dir)
def finditem(item, preferred_authorname, library='eBook'): """ Try to find book matching the csv item in the database Return database entry, or False if not found """ myDB = database.DBConnection() bookmatch = "" isbn10 = "" isbn13 = "" bookid = "" bookname = item['Title'] bookname = makeUnicode(bookname) if 'ISBN' in item: isbn10 = item['ISBN'] if 'ISBN13' in item: isbn13 = item['ISBN13'] if 'BookID' in item: bookid = item['BookID'] # try to find book in our database using bookid or isbn, or if that fails, name matching cmd = 'SELECT AuthorName,BookName,BookID,books.Status,AudioStatus,Requester,' cmd += 'AudioRequester FROM books,authors where books.AuthorID = authors.AuthorID ' if bookid: fullcmd = cmd + 'and BookID=?' bookmatch = myDB.match(fullcmd, (bookid, )) if not bookmatch: if is_valid_isbn(isbn10): fullcmd = cmd + 'and BookIsbn=?' bookmatch = myDB.match(fullcmd, (isbn10, )) if not bookmatch: if is_valid_isbn(isbn13): fullcmd = cmd + 'and BookIsbn=?' bookmatch = myDB.match(fullcmd, (isbn13, )) if not bookmatch: bookid, mtype = find_book_in_db(preferred_authorname, bookname, ignored=False, library=library) if bookid: fullcmd = cmd + 'and BookID=?' bookmatch = myDB.match(fullcmd, (bookid, )) return bookmatch
def finditem(item, headers): """ Try to find book matching the csv item in the database Return database entry, or False if not found """ myDB = database.DBConnection() bookmatch = False isbn10 = "" isbn13 = "" bookid = "" bookname = item['Title'] authorname = item['Author'] if hasattr(authorname, 'decode'): authorname = authorname.decode(lazylibrarian.SYS_ENCODING) if hasattr(bookname, 'decode'): bookname = bookname.decode(lazylibrarian.SYS_ENCODING) if 'ISBN' in headers: isbn10 = item['ISBN'] if 'ISBN13' in headers: isbn13 = item['ISBN13'] if 'BookID' in headers: bookid = item['BookID'] # try to find book in our database using bookid or isbn, or if that fails, name matching if bookid: bookmatch = myDB.action('SELECT * FROM books where BookID=%s' % (bookid)).fetchone() if not bookmatch: if is_valid_isbn(isbn10): bookmatch = myDB.action('SELECT * FROM books where BookIsbn=%s' % (isbn10)).fetchone() if not bookmatch: if is_valid_isbn(isbn13): bookmatch = myDB.action('SELECT * FROM books where BookIsbn=%s' % (isbn13)).fetchone() if not bookmatch: bookid = find_book_in_db(myDB, authorname, bookname) if bookid: bookmatch = myDB.action('SELECT * FROM books where BookID="%s"' % (bookid)).fetchone() return bookmatch
def finditem(item, authorname, headers): """ Try to find book matching the csv item in the database Return database entry, or False if not found """ myDB = database.DBConnection() bookmatch = "" isbn10 = "" isbn13 = "" bookid = "" bookname = item['Title'] if isinstance(bookname, str) and hasattr(bookname, "decode"): bookname = bookname.decode(lazylibrarian.SYS_ENCODING) if 'ISBN' in headers: isbn10 = item['ISBN'] if 'ISBN13' in headers: isbn13 = item['ISBN13'] if 'BookID' in headers: bookid = item['BookID'] # try to find book in our database using bookid or isbn, or if that fails, name matching cmd = 'SELECT AuthorName,BookName,BookID,books.Status FROM books,authors where books.AuthorID = authors.AuthorID ' if bookid: fullcmd = cmd + 'and BookID=?' bookmatch = myDB.match(fullcmd, (bookid,)) if not bookmatch: if is_valid_isbn(isbn10): fullcmd = cmd + 'and BookIsbn=?' bookmatch = myDB.match(fullcmd, (isbn10,)) if not bookmatch: if is_valid_isbn(isbn13): fullcmd = cmd + 'and BookIsbn=?' bookmatch = myDB.match(fullcmd, (isbn13,)) if not bookmatch: bookid = find_book_in_db(authorname, bookname) if bookid: fullcmd = cmd + 'and BookID=?' bookmatch = myDB.match(fullcmd, (bookid,)) return bookmatch
def finditem(item, preferred_authorname, library='eBook'): """ Try to find book matching the csv item in the database Return database entry, or False if not found """ myDB = database.DBConnection() bookmatch = "" isbn10 = "" isbn13 = "" bookid = "" bookname = item['Title'] bookname = makeUnicode(bookname) if 'ISBN' in item: isbn10 = item['ISBN'] if 'ISBN13' in item: isbn13 = item['ISBN13'] if 'BookID' in item: bookid = item['BookID'] # try to find book in our database using bookid or isbn, or if that fails, name matching cmd = 'SELECT AuthorName,BookName,BookID,books.Status,AudioStatus,Requester,' cmd += 'AudioRequester FROM books,authors where books.AuthorID = authors.AuthorID ' if bookid: fullcmd = cmd + 'and BookID=?' bookmatch = myDB.match(fullcmd, (bookid,)) if not bookmatch: if is_valid_isbn(isbn10): fullcmd = cmd + 'and BookIsbn=?' bookmatch = myDB.match(fullcmd, (isbn10,)) if not bookmatch: if is_valid_isbn(isbn13): fullcmd = cmd + 'and BookIsbn=?' bookmatch = myDB.match(fullcmd, (isbn13,)) if not bookmatch: bookid, mtype = find_book_in_db(preferred_authorname, bookname, ignored=False, library=library) if bookid: fullcmd = cmd + 'and BookID=?' bookmatch = myDB.match(fullcmd, (bookid,)) return bookmatch
def processAlternate(source_dir=None): # import a book from an alternate directory try: if not source_dir: logger.warn("Alternate Directory not configured") return False elif not os.path.isdir(source_dir): logger.warn("Alternate Directory [%s] not found" % source_dir) return False if source_dir == lazylibrarian.DIRECTORY('Destination'): logger.warn('Alternate directory must not be the same as Destination') return False logger.debug('Processing alternate directory %s' % source_dir) # first, recursively process any books in subdirectories for fname in os.listdir(source_dir): subdir = os.path.join(source_dir, fname) if os.path.isdir(subdir): processAlternate(subdir) # only import one book from each alternate (sub)directory, this is because # the importer may delete the directory after importing a book, # depending on lazylibrarian.DESTINATION_COPY setting # also if multiple books in a folder and only a "metadata.opf" # which book is it for? new_book = book_file(source_dir, booktype='book') if new_book: metadata = {} # see if there is a metadata file in this folder with the info we need # try book_name.opf first, or fall back to any filename.opf metafile = os.path.splitext(new_book)[0] + '.opf' if not os.path.isfile(metafile): metafile = opf_file(source_dir) if metafile and os.path.isfile(metafile): try: metadata = get_book_info(metafile) except Exception as e: logger.debug('Failed to read metadata from %s, %s' % (metafile, str(e))) else: logger.debug('No metadata file found for %s' % new_book) if 'title' not in metadata or 'creator' not in metadata: # if not got both, try to get metadata from the book file try: metadata = get_book_info(new_book) except Exception as e: logger.debug('No metadata found in %s, %s' % (new_book, str(e))) if 'title' in metadata and 'creator' in metadata: authorname = metadata['creator'] bookname = metadata['title'] myDB = database.DBConnection() authmatch = myDB.match('SELECT * FROM authors where AuthorName="%s"' % (authorname)) if not authmatch: # try goodreads preferred authorname logger.debug("Checking GoodReads for [%s]" % authorname) GR = GoodReads(authorname) try: author_gr = GR.find_author_id() except Exception: logger.debug("No author id for [%s]" % authorname) if author_gr: grauthorname = author_gr['authorname'] logger.debug("GoodReads reports [%s] for [%s]" % (grauthorname, authorname)) authorname = grauthorname authmatch = myDB.match('SELECT * FROM authors where AuthorName="%s"' % (authorname)) if authmatch: logger.debug("ALT: Author %s found in database" % (authorname)) else: logger.debug("ALT: Author %s not found, adding to database" % (authorname)) addAuthorToDB(authorname) bookid = find_book_in_db(myDB, authorname, bookname) if bookid: return import_book(source_dir, bookid) else: logger.warn("Book %s by %s not found in database" % (bookname, authorname)) else: logger.warn('Book %s has no metadata, unable to import' % new_book) else: logger.warn("No book file found in %s" % source_dir) return False except Exception as e: logger.error('Unhandled exception in processAlternate: %s' % traceback.format_exc())
def syncCalibreList(col_read=None, col_toread=None, userid=None): """ Get the lazylibrarian bookid for each read/toread calibre book so we can map our id to theirs, and sync current/supplied user's read/toread or supplied read/toread columns to calibre database. Return message giving totals """ myDB = database.DBConnection() if not userid: cookie = cherrypy.request.cookie if cookie and 'll_uid' in cookie.keys(): userid = cookie['ll_uid'].value if userid: res = myDB.match( 'SELECT UserName,ToRead,HaveRead,CalibreRead,CalibreToRead,Perms from users where UserID=?', (userid, )) if res: username = res['UserName'] if not col_read: col_read = res['CalibreRead'] if not col_toread: col_toread = res['CalibreToRead'] toreadlist = getList(res['ToRead']) readlist = getList(res['HaveRead']) # suppress duplicates (just in case) toreadlist = list(set(toreadlist)) readlist = list(set(readlist)) else: return "Error: Unable to get user column settings for %s" % userid if not userid: return "Error: Unable to find current userid" if not col_read and not col_toread: return "User %s has no calibre columns set" % username # check user columns exist in calibre and create if not res = calibredb('custom_columns') columns = res[0].split('\n') custom_columns = [] for column in columns: if column: custom_columns.append(column.split(' (')[0]) if col_read not in custom_columns: added = calibredb('add_custom_column', [col_read, col_read, 'bool']) if "column created" not in added[0]: return added if col_toread not in custom_columns: added = calibredb('add_custom_column', [col_toread, col_toread, 'bool']) if "column created" not in added[0]: return added nomatch = 0 readcol = '' toreadcol = '' map_ctol = {} map_ltoc = {} if col_read: readcol = '*' + col_read if col_toread: toreadcol = '*' + col_toread calibre_list = calibreList(col_read, col_toread) if not isinstance(calibre_list, list): # got an error message from calibredb return '"%s"' % calibre_list for item in calibre_list: if toreadcol and toreadcol in item or readcol and readcol in item: authorname, authorid, added = addAuthorNameToDB(item['authors'], refresh=False, addbooks=False) if authorname: if authorname != item['authors']: logger.debug( "Changed authorname for [%s] from [%s] to [%s]" % (item['title'], item['authors'], authorname)) item['authors'] = authorname bookid = find_book_in_db(authorname, item['title']) if not bookid: searchterm = "%s <ll> %s" % (item['title'], authorname) results = search_for(unaccented(searchterm)) if results: result = results[0] if result['author_fuzz'] > lazylibrarian.CONFIG['MATCH_RATIO'] \ and result['book_fuzz'] > lazylibrarian.CONFIG['MATCH_RATIO']: logger.debug( "Found (%s%% %s%%) %s: %s" % (result['author_fuzz'], result['book_fuzz'], result['authorname'], result['bookname'])) bookid = result['bookid'] import_book(bookid) if bookid: # NOTE: calibre bookid is always an integer, lazylibrarian bookid is a string # (goodreads could be used as an int, but googlebooks can't as it's alphanumeric) # so convert all dict items to strings for ease of matching. map_ctol[str(item['id'])] = str(bookid) map_ltoc[str(bookid)] = str(item['id']) else: logger.warn( 'Calibre Book [%s] by [%s] is not in lazylibrarian database' % (item['title'], authorname)) nomatch += 1 else: logger.warn( 'Calibre Author [%s] not matched in lazylibrarian database' % (item['authors'])) nomatch += 1 # Now check current users lazylibrarian read/toread against the calibre library, warn about missing ones # which might be books calibre doesn't have, or might be minor differences in author or title for idlist in [("Read", readlist), ("To_Read", toreadlist)]: booklist = idlist[1] for bookid in booklist: cmd = "SELECT AuthorID,BookName from books where BookID=?" book = myDB.match(cmd, (bookid, )) if not book: logger.error('Error finding bookid %s' % bookid) else: cmd = "SELECT AuthorName from authors where AuthorID=?" author = myDB.match(cmd, (book['AuthorID'], )) if not author: logger.error('Error finding authorid %s' % book['AuthorID']) else: match = False for item in calibre_list: if item['authors'] == author['AuthorName'] and item[ 'title'] == book['BookName']: logger.debug("Exact match for %s [%s]" % (idlist[0], book['BookName'])) map_ctol[str(item['id'])] = str(bookid) map_ltoc[str(bookid)] = str(item['id']) match = True break if not match: high = 0 highname = '' highid = '' for item in calibre_list: if item['authors'] == author['AuthorName']: n = fuzz.token_sort_ratio( item['title'], book['BookName']) if n > high: high = n highname = item['title'] highid = item['id'] if high > 95: logger.debug( "Found ratio match %s%% [%s] for %s [%s]" % (high, highname, idlist[0], book['BookName'])) map_ctol[str(highid)] = str(bookid) map_ltoc[str(bookid)] = str(highid) match = True if not match: logger.warn( "No match for %s %s by %s in calibre database, closest match %s%% [%s]" % (idlist[0], book['BookName'], author['AuthorName'], high, highname)) nomatch += 1 logger.debug("BookID mapping complete, %s match %s, nomatch %s" % (username, len(map_ctol), nomatch)) # now sync the lists if userid: last_read = [] last_toread = [] calibre_read = [] calibre_toread = [] cmd = 'select SyncList from sync where UserID=? and Label=?' res = myDB.match(cmd, (userid, col_read)) if res: last_read = getList(res['SyncList']) res = myDB.match(cmd, (userid, col_toread)) if res: last_toread = getList(res['SyncList']) for item in calibre_list: if toreadcol and toreadcol in item and item[ toreadcol]: # only if True if str(item['id']) in map_ctol: calibre_toread.append(map_ctol[str(item['id'])]) else: logger.warn( "Calibre to_read book %s:%s has no lazylibrarian bookid" % (item['authors'], item['title'])) if readcol and readcol in item and item[readcol]: # only if True if str(item['id']) in map_ctol: calibre_read.append(map_ctol[str(item['id'])]) else: logger.warn( "Calibre read book %s:%s has no lazylibrarian bookid" % (item['authors'], item['title'])) logger.debug("Found %s calibre read, %s calibre toread" % (len(calibre_read), len(calibre_toread))) logger.debug("Found %s lazylib read, %s lazylib toread" % (len(readlist), len(toreadlist))) added_to_ll_toread = list(set(toreadlist) - set(last_toread)) removed_from_ll_toread = list(set(last_toread) - set(toreadlist)) added_to_ll_read = list(set(readlist) - set(last_read)) removed_from_ll_read = list(set(last_read) - set(readlist)) logger.debug("lazylibrarian changes to copy to calibre: %s %s %s %s" % (len(added_to_ll_toread), len(removed_from_ll_toread), len(added_to_ll_read), len(removed_from_ll_read))) added_to_calibre_toread = list(set(calibre_toread) - set(last_toread)) removed_from_calibre_toread = list( set(last_toread) - set(calibre_toread)) added_to_calibre_read = list(set(calibre_read) - set(last_read)) removed_from_calibre_read = list(set(last_read) - set(calibre_read)) logger.debug( "calibre changes to copy to lazylibrarian: %s %s %s %s" % (len(added_to_calibre_toread), len(removed_from_calibre_toread), len(added_to_calibre_read), len(removed_from_calibre_read))) calibre_changes = 0 for item in added_to_calibre_read: if item not in readlist: readlist.append(item) logger.debug("Lazylibrarian marked %s as read" % item) calibre_changes += 1 for item in added_to_calibre_toread: if item not in toreadlist: toreadlist.append(item) logger.debug("Lazylibrarian marked %s as to_read" % item) calibre_changes += 1 for item in removed_from_calibre_read: if item in readlist: readlist.remove(item) logger.debug("Lazylibrarian removed %s from read" % item) calibre_changes += 1 for item in removed_from_calibre_toread: if item in toreadlist: toreadlist.remove(item) logger.debug("Lazylibrarian removed %s from to_read" % item) calibre_changes += 1 if calibre_changes: myDB.action('UPDATE users SET ToRead=?,HaveRead=? WHERE UserID=?', (', '.join(toreadlist), ', '.join(readlist), userid)) ll_changes = 0 for item in added_to_ll_toread: if item in map_ltoc: res, err, rc = calibredb('set_custom', [col_toread, map_ltoc[item], 'true'], []) if rc: msg = "calibredb set_custom error: " if err: logger.error(msg + err) elif res: logger.error(msg + res) else: logger.error(msg + str(rc)) else: ll_changes += 1 else: logger.warn("Unable to set calibre %s true for %s" % (col_toread, item)) for item in removed_from_ll_toread: if item in map_ltoc: res, err, rc = calibredb('set_custom', [col_toread, map_ltoc[item], ''], []) if rc: msg = "calibredb set_custom error: " if err: logger.error(msg + err) elif res: logger.error(msg + res) else: logger.error(msg + str(rc)) else: ll_changes += 1 else: logger.warn("Unable to clear calibre %s for %s" % (col_toread, item)) for item in added_to_ll_read: if item in map_ltoc: res, err, rc = calibredb('set_custom', [col_read, map_ltoc[item], 'true'], []) if rc: msg = "calibredb set_custom error: " if err: logger.error(msg + err) elif res: logger.error(msg + res) else: logger.error(msg + str(rc)) else: ll_changes += 1 else: logger.warn("Unable to set calibre %s true for %s" % (col_read, item)) for item in removed_from_ll_read: if item in map_ltoc: res, err, rc = calibredb('set_custom', [col_read, map_ltoc[item], ''], []) if rc: msg = "calibredb set_custom error: " if err: logger.error(msg + err) elif res: logger.error(msg + res) else: logger.error(msg + str(rc)) else: ll_changes += 1 else: logger.warn("Unable to clear calibre %s for %s" % (col_read, item)) # store current sync list as comparison for next sync controlValueDict = {"UserID": userid, "Label": col_read} newValueDict = { "Date": str(time.time()), "Synclist": ', '.join(readlist) } myDB.upsert("sync", newValueDict, controlValueDict) controlValueDict = {"UserID": userid, "Label": col_toread} newValueDict = { "Date": str(time.time()), "Synclist": ', '.join(toreadlist) } myDB.upsert("sync", newValueDict, controlValueDict) msg = "%s sync updated: %s calibre, %s lazylibrarian" % ( username, ll_changes, calibre_changes) return msg
def processCSV(search_dir=None): """ Find a csv file in the search_dir and process all the books in it, adding authors to the database if not found, and marking the books as "Wanted" """ if not search_dir: logger.warn("Alternate Directory must not be empty") return False csvFile = csv_file(search_dir) headers = None content = {} if not csvFile: logger.warn("No CSV file found in %s" % search_dir) else: logger.debug('Reading file %s' % csvFile) reader=csv.reader(open(csvFile)) for row in reader: if reader.line_num == 1: # If we are on the first line, create the headers list from the first row # by taking a slice from item 1 as we don't need the very first header. headers = row[1:] else: # Otherwise, the key in the content dictionary is the first item in the # row and we can create the sub-dictionary by using the zip() function. content[row[0]] = dict(zip(headers, row[1:])) # We can now get to the content by using the resulting dictionary, so to see # the list of lines, we can do: #print content.keys() # to get a list of bookIDs # To see the list of fields available for each book #print headers if 'Author' not in headers or 'Title' not in headers: logger.warn('Invalid CSV file found %s' % csvFile) return myDB = database.DBConnection() bookcount = 0 authcount = 0 skipcount = 0 logger.debug("CSV: Found %s entries in csv file" % len(content.keys())) for bookid in content.keys(): authorname = content[bookid]['Author'] authmatch = myDB.action('SELECT * FROM authors where AuthorName="%s"' % (authorname)).fetchone() if authmatch: logger.debug("CSV: Author %s found in database" % (authorname)) else: logger.debug("CSV: Author %s not found, adding to database" % (authorname)) importer.addAuthorToDB(authorname) authcount = authcount + 1 bookmatch = 0 isbn10="" isbn13="" bookname = content[bookid]['Title'] if 'ISBN' in headers: isbn10 = content[bookid]['ISBN'] if 'ISBN13' in headers: isbn13 = content[bookid]['ISBN13'] # try to find book in our database using isbn, or if that fails, fuzzy name matching if formatter.is_valid_isbn(isbn10): bookmatch = myDB.action('SELECT * FROM books where Bookisbn=%s' % (isbn10)).fetchone() if not bookmatch: if formatter.is_valid_isbn(isbn13): bookmatch = myDB.action('SELECT * FROM books where BookIsbn=%s' % (isbn13)).fetchone() if not bookmatch: bookid = librarysync.find_book_in_db(myDB, authorname, bookname) if bookid: bookmatch = myDB.action('SELECT * FROM books where BookID="%s"' % (bookid)).fetchone() if bookmatch: authorname = bookmatch['AuthorName'] bookname = bookmatch['BookName'] bookid = bookmatch['BookID'] bookstatus = bookmatch['Status'] if bookstatus == 'Open' or bookstatus == 'Wanted' or bookstatus == 'Have': logger.info('Found book %s by %s, already marked as "%s"' % (bookname, authorname, bookstatus)) else: # skipped/ignored logger.info('Found book %s by %s, marking as "Wanted"' % (bookname, authorname)) controlValueDict = {"BookID": bookid} newValueDict = {"Status": "Wanted"} myDB.upsert("books", newValueDict, controlValueDict) bookcount = bookcount + 1 else: logger.warn("Skipping book %s by %s, not found in database" % (bookname, authorname)) skipcount = skipcount + 1 logger.info("Added %i new authors, marked %i books as 'Wanted', %i books not found" % (authcount, bookcount, skipcount))
def processAlternate(source_dir=None): # import a book from an alternate directory if not source_dir or os.path.isdir(source_dir) is False: logger.warn('Alternate directory not found') return if source_dir == lazylibrarian.DESTINATION_DIR: logger.warn('Alternate directory must not be the same as destination') return logger.debug('Processing alternate directory %s' % source_dir) # first, recursively process any books in subdirectories for fname in os.listdir(source_dir): subdir = os.path.join(source_dir, fname) if os.path.isdir(subdir): processAlternate(subdir) # only import one book from each alternate (sub)directory, this is because # the importer may delete the directory after importing a book, # depending on lazylibrarian.DESTINATION_COPY setting # also if multiple books in a folder and only a "metadata.opf" # which book is it for? new_book = book_file(source_dir, booktype='book') if new_book: metadata = {} # see if there is a metadata file in this folder with the info we need # try book_name.opf first, or fall back to any filename.opf metafile = os.path.splitext(new_book)[0] + '.opf' if not os.path.isfile(metafile): metafile = librarysync.opf_file(source_dir) if os.path.isfile(metafile): try: metadata = librarysync.get_book_info(metafile) except: logger.debug('Failed to read metadata from %s' % metafile) else: logger.debug('No metadata file found for %s' % new_book) if not 'title' in metadata and 'creator' in metadata: # try to get metadata from the book file try: metadata = librarysync.get_book_info(new_book) except: logger.debug('No metadata found in %s' % new_book) if 'title' in metadata and 'creator' in metadata: authorname = metadata['creator'] bookname = metadata['title'] myDB = database.DBConnection() authmatch = myDB.action('SELECT * FROM authors where AuthorName="%s"' % (authorname)).fetchone() if authmatch: logger.debug("ALT: Author %s found in database" % (authorname)) else: logger.debug("ALT: Author %s not found, adding to database" % (authorname)) importer.addAuthorToDB(authorname) bookid = librarysync.find_book_in_db(myDB, authorname, bookname) if bookid: import_book(source_dir, bookid) else: logger.warn("Book %s by %s not found in database" % (bookname, authorname)) else: logger.warn('Book %s has no metadata, unable to import' % new_book) else: logger.warn("No book file found in %s" % source_dir)
def syncCalibreList(col_read=None, col_toread=None, userid=None): """ Get the lazylibrarian bookid for each read/toread calibre book so we can map our id to theirs, and sync current/supplied user's read/toread or supplied read/toread columns to calibre database. Return message giving totals """ myDB = database.DBConnection() username = '' readlist = [] toreadlist = [] if not userid: cookie = cherrypy.request.cookie if cookie and 'll_uid' in list(cookie.keys()): userid = cookie['ll_uid'].value if userid: res = myDB.match('SELECT UserName,ToRead,HaveRead,CalibreRead,CalibreToRead,Perms from users where UserID=?', (userid,)) if res: username = res['UserName'] if not col_read: col_read = res['CalibreRead'] if not col_toread: col_toread = res['CalibreToRead'] toreadlist = getList(res['ToRead']) readlist = getList(res['HaveRead']) # suppress duplicates (just in case) toreadlist = list(set(toreadlist)) readlist = list(set(readlist)) else: return "Error: Unable to get user column settings for %s" % userid if not userid: return "Error: Unable to find current userid" if not col_read and not col_toread: return "User %s has no calibre columns set" % username # check user columns exist in calibre and create if not res = calibredb('custom_columns') columns = res[0].split('\n') custom_columns = [] for column in columns: if column: custom_columns.append(column.split(' (')[0]) if col_read not in custom_columns: added = calibredb('add_custom_column', [col_read, col_read, 'bool']) if "column created" not in added[0]: return added if col_toread not in custom_columns: added = calibredb('add_custom_column', [col_toread, col_toread, 'bool']) if "column created" not in added[0]: return added nomatch = 0 readcol = '' toreadcol = '' map_ctol = {} map_ltoc = {} if col_read: readcol = '*' + col_read if col_toread: toreadcol = '*' + col_toread calibre_list = calibreList(col_read, col_toread) if not isinstance(calibre_list, list): # got an error message from calibredb return '"%s"' % calibre_list for item in calibre_list: if toreadcol and toreadcol in item or readcol and readcol in item: authorname, authorid, added = addAuthorNameToDB(item['authors'], refresh=False, addbooks=False) if authorname: if authorname != item['authors']: logger.debug("Changed authorname for [%s] from [%s] to [%s]" % (item['title'], item['authors'], authorname)) item['authors'] = authorname bookid, mtype = find_book_in_db(authorname, item['title'], ignored=False, library='eBook') if bookid and mtype == "Ignored": logger.warn("Book %s by %s is marked Ignored in database, importing anyway" % (item['title'], authorname)) if not bookid: searchterm = "%s <ll> %s" % (item['title'], authorname) results = search_for(unaccented(searchterm)) if results: result = results[0] if result['author_fuzz'] > lazylibrarian.CONFIG['MATCH_RATIO'] \ and result['book_fuzz'] > lazylibrarian.CONFIG['MATCH_RATIO']: logger.debug("Found (%s%% %s%%) %s: %s" % (result['author_fuzz'], result['book_fuzz'], result['authorname'], result['bookname'])) bookid = result['bookid'] import_book(bookid) if bookid: # NOTE: calibre bookid is always an integer, lazylibrarian bookid is a string # (goodreads could be used as an int, but googlebooks can't as it's alphanumeric) # so convert all dict items to strings for ease of matching. map_ctol[str(item['id'])] = str(bookid) map_ltoc[str(bookid)] = str(item['id']) else: logger.warn('Calibre Book [%s] by [%s] is not in lazylibrarian database' % (item['title'], authorname)) nomatch += 1 else: logger.warn('Calibre Author [%s] not matched in lazylibrarian database' % (item['authors'])) nomatch += 1 # Now check current users lazylibrarian read/toread against the calibre library, warn about missing ones # which might be books calibre doesn't have, or might be minor differences in author or title for idlist in [("Read", readlist), ("To_Read", toreadlist)]: booklist = idlist[1] for bookid in booklist: cmd = "SELECT AuthorID,BookName from books where BookID=?" book = myDB.match(cmd, (bookid,)) if not book: logger.error('Error finding bookid %s' % bookid) else: cmd = "SELECT AuthorName from authors where AuthorID=?" author = myDB.match(cmd, (book['AuthorID'],)) if not author: logger.error('Error finding authorid %s' % book['AuthorID']) else: match = False high = 0 highname = '' for item in calibre_list: if item['authors'] == author['AuthorName'] and item['title'] == book['BookName']: logger.debug("Exact match for %s [%s]" % (idlist[0], book['BookName'])) map_ctol[str(item['id'])] = str(bookid) map_ltoc[str(bookid)] = str(item['id']) match = True break if not match: highid = '' for item in calibre_list: if item['authors'] == author['AuthorName']: n = fuzz.token_sort_ratio(item['title'], book['BookName']) if n > high: high = n highname = item['title'] highid = item['id'] if high > 95: logger.debug("Found ratio match %s%% [%s] for %s [%s]" % (high, highname, idlist[0], book['BookName'])) map_ctol[str(highid)] = str(bookid) map_ltoc[str(bookid)] = str(highid) match = True if not match: logger.warn("No match for %s %s by %s in calibre database, closest match %s%% [%s]" % (idlist[0], book['BookName'], author['AuthorName'], high, highname)) nomatch += 1 logger.debug("BookID mapping complete, %s match %s, nomatch %s" % (username, len(map_ctol), nomatch)) # now sync the lists if not userid: msg = "No userid found" else: last_read = [] last_toread = [] calibre_read = [] calibre_toread = [] cmd = 'select SyncList from sync where UserID=? and Label=?' res = myDB.match(cmd, (userid, col_read)) if res: last_read = getList(res['SyncList']) res = myDB.match(cmd, (userid, col_toread)) if res: last_toread = getList(res['SyncList']) for item in calibre_list: if toreadcol and toreadcol in item and item[toreadcol]: # only if True if str(item['id']) in map_ctol: calibre_toread.append(map_ctol[str(item['id'])]) else: logger.warn("Calibre to_read book %s:%s has no lazylibrarian bookid" % (item['authors'], item['title'])) if readcol and readcol in item and item[readcol]: # only if True if str(item['id']) in map_ctol: calibre_read.append(map_ctol[str(item['id'])]) else: logger.warn("Calibre read book %s:%s has no lazylibrarian bookid" % (item['authors'], item['title'])) logger.debug("Found %s calibre read, %s calibre toread" % (len(calibre_read), len(calibre_toread))) logger.debug("Found %s lazylib read, %s lazylib toread" % (len(readlist), len(toreadlist))) added_to_ll_toread = list(set(toreadlist) - set(last_toread)) removed_from_ll_toread = list(set(last_toread) - set(toreadlist)) added_to_ll_read = list(set(readlist) - set(last_read)) removed_from_ll_read = list(set(last_read) - set(readlist)) logger.debug("lazylibrarian changes to copy to calibre: %s %s %s %s" % (len(added_to_ll_toread), len(removed_from_ll_toread), len(added_to_ll_read), len(removed_from_ll_read))) added_to_calibre_toread = list(set(calibre_toread) - set(last_toread)) removed_from_calibre_toread = list(set(last_toread) - set(calibre_toread)) added_to_calibre_read = list(set(calibre_read) - set(last_read)) removed_from_calibre_read = list(set(last_read) - set(calibre_read)) logger.debug("calibre changes to copy to lazylibrarian: %s %s %s %s" % (len(added_to_calibre_toread), len(removed_from_calibre_toread), len(added_to_calibre_read), len(removed_from_calibre_read))) calibre_changes = 0 for item in added_to_calibre_read: if item not in readlist: readlist.append(item) logger.debug("Lazylibrarian marked %s as read" % item) calibre_changes += 1 for item in added_to_calibre_toread: if item not in toreadlist: toreadlist.append(item) logger.debug("Lazylibrarian marked %s as to_read" % item) calibre_changes += 1 for item in removed_from_calibre_read: if item in readlist: readlist.remove(item) logger.debug("Lazylibrarian removed %s from read" % item) calibre_changes += 1 for item in removed_from_calibre_toread: if item in toreadlist: toreadlist.remove(item) logger.debug("Lazylibrarian removed %s from to_read" % item) calibre_changes += 1 if calibre_changes: myDB.action('UPDATE users SET ToRead=?,HaveRead=? WHERE UserID=?', (', '.join(toreadlist), ', '.join(readlist), userid)) ll_changes = 0 for item in added_to_ll_toread: if item in map_ltoc: res, err, rc = calibredb('set_custom', [col_toread, map_ltoc[item], 'true'], []) if rc: msg = "calibredb set_custom error: " if err: logger.error(msg + err) elif res: logger.error(msg + res) else: logger.error(msg + str(rc)) else: ll_changes += 1 else: logger.warn("Unable to set calibre %s true for %s" % (col_toread, item)) for item in removed_from_ll_toread: if item in map_ltoc: res, err, rc = calibredb('set_custom', [col_toread, map_ltoc[item], ''], []) if rc: msg = "calibredb set_custom error: " if err: logger.error(msg + err) elif res: logger.error(msg + res) else: logger.error(msg + str(rc)) else: ll_changes += 1 else: logger.warn("Unable to clear calibre %s for %s" % (col_toread, item)) for item in added_to_ll_read: if item in map_ltoc: res, err, rc = calibredb('set_custom', [col_read, map_ltoc[item], 'true'], []) if rc: msg = "calibredb set_custom error: " if err: logger.error(msg + err) elif res: logger.error(msg + res) else: logger.error(msg + str(rc)) else: ll_changes += 1 else: logger.warn("Unable to set calibre %s true for %s" % (col_read, item)) for item in removed_from_ll_read: if item in map_ltoc: res, err, rc = calibredb('set_custom', [col_read, map_ltoc[item], ''], []) if rc: msg = "calibredb set_custom error: " if err: logger.error(msg + err) elif res: logger.error(msg + res) else: logger.error(msg + str(rc)) else: ll_changes += 1 else: logger.warn("Unable to clear calibre %s for %s" % (col_read, item)) # store current sync list as comparison for next sync controlValueDict = {"UserID": userid, "Label": col_read} newValueDict = {"Date": str(time.time()), "Synclist": ', '.join(readlist)} myDB.upsert("sync", newValueDict, controlValueDict) controlValueDict = {"UserID": userid, "Label": col_toread} newValueDict = {"Date": str(time.time()), "Synclist": ', '.join(toreadlist)} myDB.upsert("sync", newValueDict, controlValueDict) msg = "%s sync updated: %s calibre, %s lazylibrarian" % (username, ll_changes, calibre_changes) return msg
def processCSV(search_dir=None): """ Find a csv file in the search_dir and process all the books in it, adding authors to the database if not found, and marking the books as "Wanted" """ if not search_dir or os.path.isdir(search_dir) is False: logger.warn(u"Alternate Directory must not be empty") return False csvFile = csv_file(search_dir) headers = None content = {} if not csvFile: logger.warn(u"No CSV file found in %s" % search_dir) else: logger.debug(u'Reading file %s' % csvFile) reader = csv.reader(open(csvFile)) for row in reader: if reader.line_num == 1: # If we are on the first line, create the headers list from the first row # by taking a slice from item 1 as we don't need the very first header. headers = row[1:] else: # Otherwise, the key in the content dictionary is the first item in the # row and we can create the sub-dictionary by using the zip() function. content[row[0]] = dict(zip(headers, row[1:])) # We can now get to the content by using the resulting dictionary, so to see # the list of lines, we can do: # print content.keys() # to get a list of bookIDs # To see the list of fields available for each book # print headers if 'Author' not in headers or 'Title' not in headers: logger.warn(u'Invalid CSV file found %s' % csvFile) return myDB = database.DBConnection() bookcount = 0 authcount = 0 skipcount = 0 logger.debug(u"CSV: Found %s entries in csv file" % len(content.keys())) for bookid in content.keys(): authorname = formatter.latinToAscii(content[bookid]['Author']) authmatch = myDB.action('SELECT * FROM authors where AuthorName="%s"' % (authorname)).fetchone() if authmatch: logger.debug(u"CSV: Author %s found in database" % (authorname)) else: logger.debug(u"CSV: Author %s not found, adding to database" % (authorname)) importer.addAuthorToDB(authorname) authcount = authcount + 1 bookmatch = 0 isbn10 = "" isbn13 = "" bookname = formatter.latinToAscii(content[bookid]['Title']) if 'ISBN' in headers: isbn10 = content[bookid]['ISBN'] if 'ISBN13' in headers: isbn13 = content[bookid]['ISBN13'] # try to find book in our database using isbn, or if that fails, name matching if formatter.is_valid_isbn(isbn10): bookmatch = myDB.action('SELECT * FROM books where Bookisbn=%s' % (isbn10)).fetchone() if not bookmatch: if formatter.is_valid_isbn(isbn13): bookmatch = myDB.action('SELECT * FROM books where BookIsbn=%s' % (isbn13)).fetchone() if not bookmatch: bookid = librarysync.find_book_in_db(myDB, authorname, bookname) if bookid: bookmatch = myDB.action('SELECT * FROM books where BookID="%s"' % (bookid)).fetchone() if bookmatch: authorname = bookmatch['AuthorName'] bookname = bookmatch['BookName'] bookid = bookmatch['BookID'] bookstatus = bookmatch['Status'] if bookstatus == 'Open' or bookstatus == 'Wanted' or bookstatus == 'Have': logger.info(u'Found book %s by %s, already marked as "%s"' % (bookname, authorname, bookstatus)) else: # skipped/ignored logger.info(u'Found book %s by %s, marking as "Wanted"' % (bookname, authorname)) controlValueDict = {"BookID": bookid} newValueDict = {"Status": "Wanted"} myDB.upsert("books", newValueDict, controlValueDict) bookcount = bookcount + 1 else: logger.warn(u"Skipping book %s by %s, not found in database" % (bookname, authorname)) skipcount = skipcount + 1 logger.info(u"Added %i new authors, marked %i books as 'Wanted', %i books not found" % (authcount, bookcount, skipcount))