def finditem(item, preferred_authorname):
"""
Try to find book matching the csv item in the database
Return database entry, or False if not found
"""
myDB = database.DBConnection()
bookmatch = ""
isbn10 = ""
isbn13 = ""
bookid = ""
bookname = item['Title']
bookname = makeUnicode(bookname)
if 'ISBN' in item:
isbn10 = item['ISBN']
if 'ISBN13' in item:
isbn13 = item['ISBN13']
if 'BookID' in item:
bookid = item['BookID']
# try to find book in our database using bookid or isbn, or if that fails, name matching
cmd = 'SELECT AuthorName,BookName,BookID,books.Status FROM books,authors where books.AuthorID = authors.AuthorID '
if bookid:
fullcmd = cmd + 'and BookID=?'
bookmatch = myDB.match(fullcmd, (bookid,))
if not bookmatch:
if is_valid_isbn(isbn10):
fullcmd = cmd + 'and BookIsbn=?'
bookmatch = myDB.match(fullcmd, (isbn10,))
if not bookmatch:
if is_valid_isbn(isbn13):
fullcmd = cmd + 'and BookIsbn=?'
bookmatch = myDB.match(fullcmd, (isbn13,))
if not bookmatch:
bookid = find_book_in_db(preferred_authorname, bookname, ignored=False)
if not bookid:
bookid = find_book_in_db(preferred_authorname, bookname, ignored=True)
if bookid:
logger.warn("Book %s by %s is marked Ignored in database, importing anyway" %
(bookname, preferred_authorname))
if bookid:
fullcmd = cmd + 'and BookID=?'
bookmatch = myDB.match(fullcmd, (bookid,))
return bookmatch
def processAlternate(source_dir=None):
# import a book from an alternate directory
if not source_dir or os.path.isdir(source_dir) is False:
logger.warn('Alternate directory must not be empty')
return
if source_dir == lazylibrarian.DESTINATION_DIR:
logger.warn('Alternate directory must not be the same as destination')
return
new_book = book_file(source_dir, booktype='book')
if new_book:
# see if there is a metadata file in this folder with the info we need
metafile = librarysync.opf_file(source_dir)
try:
metadata = librarysync.get_book_info(metafile)
except:
metadata = {}
if 'title' in metadata and 'creator' in metadata:
authorname = metadata['creator']
bookname = metadata['title']
# if not, try to get metadata from the book file
else:
try:
metadata = librarysync.get_book_info(new_book)
except:
metadata = {}
if 'title' in metadata and 'creator' in metadata:
authorname = metadata['creator']
bookname = metadata['title']
myDB = database.DBConnection()
authmatch = myDB.action(
'SELECT * FROM authors where AuthorName="%s"' %
(authorname)).fetchone()
if authmatch:
logger.debug("ALT: Author %s found in database" % (authorname))
else:
logger.debug("ALT: Author %s not found, adding to database" %
(authorname))
importer.addAuthorToDB(authorname)
bookid = librarysync.find_book_in_db(myDB, authorname, bookname)
if bookid:
import_book(source_dir, bookid)
else:
logger.warn("Book %s by %s not found in database" %
(bookname, authorname))
else:
logger.warn('Book %s has no metadata, unable to import' % new_book)
else:
logger.warn("No book file found in %s" % source_dir)
def processAlternate(source_dir=None):
# import a book from an alternate directory
if source_dir == None or source_dir == "":
logger.warn('Alternate directory must not be empty')
return
if source_dir == lazylibrarian.DESTINATION_DIR:
logger.warn('Alternate directory must not be the same as destination')
return
new_book = book_file(source_dir)
if new_book:
# see if there is a metadata file in this folder with the info we need
metafile = librarysync.opf_file(source_dir)
try:
metadata = librarysync.get_book_info(metafile)
except:
metadata = {}
if 'title' in metadata and 'creator' in metadata:
authorname = metadata['creator']
bookname = metadata['title']
# if not, try to get metadata from the book file
else:
try:
metadata = librarysync.get_book_info(new_book)
except:
metadata = {}
if 'title' in metadata and 'creator' in metadata:
authorname = metadata['creator']
bookname = metadata['title']
myDB = database.DBConnection()
authmatch = myDB.action('SELECT * FROM authors where AuthorName="%s"' % (authorname)).fetchone()
if authmatch:
logger.debug("ALT: Author %s found in database" % (authorname))
else:
logger.debug("ALT: Author %s not found, adding to database" % (authorname))
importer.addAuthorToDB(authorname)
bookid = librarysync.find_book_in_db(myDB, authorname, bookname)
if bookid:
import_book(source_dir, bookid)
else:
logger.warn("Book %s by %s not found in database" % (bookname, authorname))
else:
logger.warn('Book %s has no metadata, unable to import' % new_book)
else:
logger.warn("No book file found in %s" % source_dir)
def finditem(item, preferred_authorname, library='eBook'):
"""
Try to find book matching the csv item in the database
Return database entry, or False if not found
"""
myDB = database.DBConnection()
bookmatch = ""
isbn10 = ""
isbn13 = ""
bookid = ""
bookname = item['Title']
bookname = makeUnicode(bookname)
if 'ISBN' in item:
isbn10 = item['ISBN']
if 'ISBN13' in item:
isbn13 = item['ISBN13']
if 'BookID' in item:
bookid = item['BookID']
# try to find book in our database using bookid or isbn, or if that fails, name matching
cmd = 'SELECT AuthorName,BookName,BookID,books.Status,AudioStatus,Requester,'
cmd += 'AudioRequester FROM books,authors where books.AuthorID = authors.AuthorID '
if bookid:
fullcmd = cmd + 'and BookID=?'
bookmatch = myDB.match(fullcmd, (bookid, ))
if not bookmatch:
if is_valid_isbn(isbn10):
fullcmd = cmd + 'and BookIsbn=?'
bookmatch = myDB.match(fullcmd, (isbn10, ))
if not bookmatch:
if is_valid_isbn(isbn13):
fullcmd = cmd + 'and BookIsbn=?'
bookmatch = myDB.match(fullcmd, (isbn13, ))
if not bookmatch:
bookid, mtype = find_book_in_db(preferred_authorname,
bookname,
ignored=False,
library=library)
if bookid:
fullcmd = cmd + 'and BookID=?'
bookmatch = myDB.match(fullcmd, (bookid, ))
return bookmatch
def finditem(item, headers):
"""
Try to find book matching the csv item in the database
Return database entry, or False if not found
"""
myDB = database.DBConnection()
bookmatch = False
isbn10 = ""
isbn13 = ""
bookid = ""
bookname = item['Title']
authorname = item['Author']
if hasattr(authorname, 'decode'):
authorname = authorname.decode(lazylibrarian.SYS_ENCODING)
if hasattr(bookname, 'decode'):
bookname = bookname.decode(lazylibrarian.SYS_ENCODING)
if 'ISBN' in headers:
isbn10 = item['ISBN']
if 'ISBN13' in headers:
isbn13 = item['ISBN13']
if 'BookID' in headers:
bookid = item['BookID']
# try to find book in our database using bookid or isbn, or if that fails, name matching
if bookid:
bookmatch = myDB.action('SELECT * FROM books where BookID=%s' %
(bookid)).fetchone()
if not bookmatch:
if is_valid_isbn(isbn10):
bookmatch = myDB.action('SELECT * FROM books where BookIsbn=%s' %
(isbn10)).fetchone()
if not bookmatch:
if is_valid_isbn(isbn13):
bookmatch = myDB.action('SELECT * FROM books where BookIsbn=%s' %
(isbn13)).fetchone()
if not bookmatch:
bookid = find_book_in_db(myDB, authorname, bookname)
if bookid:
bookmatch = myDB.action('SELECT * FROM books where BookID="%s"' %
(bookid)).fetchone()
return bookmatch
def finditem(item, authorname, headers):
"""
Try to find book matching the csv item in the database
Return database entry, or False if not found
"""
myDB = database.DBConnection()
bookmatch = ""
isbn10 = ""
isbn13 = ""
bookid = ""
bookname = item['Title']
if isinstance(bookname, str) and hasattr(bookname, "decode"):
bookname = bookname.decode(lazylibrarian.SYS_ENCODING)
if 'ISBN' in headers:
isbn10 = item['ISBN']
if 'ISBN13' in headers:
isbn13 = item['ISBN13']
if 'BookID' in headers:
bookid = item['BookID']
# try to find book in our database using bookid or isbn, or if that fails, name matching
cmd = 'SELECT AuthorName,BookName,BookID,books.Status FROM books,authors where books.AuthorID = authors.AuthorID '
if bookid:
fullcmd = cmd + 'and BookID=?'
bookmatch = myDB.match(fullcmd, (bookid,))
if not bookmatch:
if is_valid_isbn(isbn10):
fullcmd = cmd + 'and BookIsbn=?'
bookmatch = myDB.match(fullcmd, (isbn10,))
if not bookmatch:
if is_valid_isbn(isbn13):
fullcmd = cmd + 'and BookIsbn=?'
bookmatch = myDB.match(fullcmd, (isbn13,))
if not bookmatch:
bookid = find_book_in_db(authorname, bookname)
if bookid:
fullcmd = cmd + 'and BookID=?'
bookmatch = myDB.match(fullcmd, (bookid,))
return bookmatch
def finditem(item, preferred_authorname, library='eBook'):
"""
Try to find book matching the csv item in the database
Return database entry, or False if not found
"""
myDB = database.DBConnection()
bookmatch = ""
isbn10 = ""
isbn13 = ""
bookid = ""
bookname = item['Title']
bookname = makeUnicode(bookname)
if 'ISBN' in item:
isbn10 = item['ISBN']
if 'ISBN13' in item:
isbn13 = item['ISBN13']
if 'BookID' in item:
bookid = item['BookID']
# try to find book in our database using bookid or isbn, or if that fails, name matching
cmd = 'SELECT AuthorName,BookName,BookID,books.Status,AudioStatus,Requester,'
cmd += 'AudioRequester FROM books,authors where books.AuthorID = authors.AuthorID '
if bookid:
fullcmd = cmd + 'and BookID=?'
bookmatch = myDB.match(fullcmd, (bookid,))
if not bookmatch:
if is_valid_isbn(isbn10):
fullcmd = cmd + 'and BookIsbn=?'
bookmatch = myDB.match(fullcmd, (isbn10,))
if not bookmatch:
if is_valid_isbn(isbn13):
fullcmd = cmd + 'and BookIsbn=?'
bookmatch = myDB.match(fullcmd, (isbn13,))
if not bookmatch:
bookid, mtype = find_book_in_db(preferred_authorname, bookname, ignored=False, library=library)
if bookid:
fullcmd = cmd + 'and BookID=?'
bookmatch = myDB.match(fullcmd, (bookid,))
return bookmatch
def finditem(item, headers):
"""
Try to find book matching the csv item in the database
Return database entry, or False if not found
"""
myDB = database.DBConnection()
bookmatch = False
isbn10 = ""
isbn13 = ""
bookid = ""
bookname = item['Title']
authorname = item['Author']
if hasattr(authorname, 'decode'):
authorname = authorname.decode(lazylibrarian.SYS_ENCODING)
if hasattr(bookname, 'decode'):
bookname = bookname.decode(lazylibrarian.SYS_ENCODING)
if 'ISBN' in headers:
isbn10 = item['ISBN']
if 'ISBN13' in headers:
isbn13 = item['ISBN13']
if 'BookID' in headers:
bookid = item['BookID']
# try to find book in our database using bookid or isbn, or if that fails, name matching
if bookid:
bookmatch = myDB.action('SELECT * FROM books where BookID=%s' % (bookid)).fetchone()
if not bookmatch:
if is_valid_isbn(isbn10):
bookmatch = myDB.action('SELECT * FROM books where BookIsbn=%s' % (isbn10)).fetchone()
if not bookmatch:
if is_valid_isbn(isbn13):
bookmatch = myDB.action('SELECT * FROM books where BookIsbn=%s' % (isbn13)).fetchone()
if not bookmatch:
bookid = find_book_in_db(myDB, authorname, bookname)
if bookid:
bookmatch = myDB.action('SELECT * FROM books where BookID="%s"' % (bookid)).fetchone()
return bookmatch
def processAlternate(source_dir=None):
# import a book from an alternate directory
try:
if not source_dir:
logger.warn("Alternate Directory not configured")
return False
elif not os.path.isdir(source_dir):
logger.warn("Alternate Directory [%s] not found" % source_dir)
return False
if source_dir == lazylibrarian.DIRECTORY('Destination'):
logger.warn('Alternate directory must not be the same as Destination')
return False
logger.debug('Processing alternate directory %s' % source_dir)
# first, recursively process any books in subdirectories
for fname in os.listdir(source_dir):
subdir = os.path.join(source_dir, fname)
if os.path.isdir(subdir):
processAlternate(subdir)
# only import one book from each alternate (sub)directory, this is because
# the importer may delete the directory after importing a book,
# depending on lazylibrarian.DESTINATION_COPY setting
# also if multiple books in a folder and only a "metadata.opf"
# which book is it for?
new_book = book_file(source_dir, booktype='book')
if new_book:
metadata = {}
# see if there is a metadata file in this folder with the info we need
# try book_name.opf first, or fall back to any filename.opf
metafile = os.path.splitext(new_book)[0] + '.opf'
if not os.path.isfile(metafile):
metafile = opf_file(source_dir)
if metafile and os.path.isfile(metafile):
try:
metadata = get_book_info(metafile)
except Exception as e:
logger.debug('Failed to read metadata from %s, %s' % (metafile, str(e)))
else:
logger.debug('No metadata file found for %s' % new_book)
if 'title' not in metadata or 'creator' not in metadata:
# if not got both, try to get metadata from the book file
try:
metadata = get_book_info(new_book)
except Exception as e:
logger.debug('No metadata found in %s, %s' % (new_book, str(e)))
if 'title' in metadata and 'creator' in metadata:
authorname = metadata['creator']
bookname = metadata['title']
myDB = database.DBConnection()
authmatch = myDB.match('SELECT * FROM authors where AuthorName="%s"' % (authorname))
if not authmatch:
# try goodreads preferred authorname
logger.debug("Checking GoodReads for [%s]" % authorname)
GR = GoodReads(authorname)
try:
author_gr = GR.find_author_id()
except Exception:
logger.debug("No author id for [%s]" % authorname)
if author_gr:
grauthorname = author_gr['authorname']
logger.debug("GoodReads reports [%s] for [%s]" % (grauthorname, authorname))
authorname = grauthorname
authmatch = myDB.match('SELECT * FROM authors where AuthorName="%s"' % (authorname))
if authmatch:
logger.debug("ALT: Author %s found in database" % (authorname))
else:
logger.debug("ALT: Author %s not found, adding to database" % (authorname))
addAuthorToDB(authorname)
bookid = find_book_in_db(myDB, authorname, bookname)
if bookid:
return import_book(source_dir, bookid)
else:
logger.warn("Book %s by %s not found in database" % (bookname, authorname))
else:
logger.warn('Book %s has no metadata, unable to import' % new_book)
else:
logger.warn("No book file found in %s" % source_dir)
return False
except Exception as e:
logger.error('Unhandled exception in processAlternate: %s' % traceback.format_exc())
def syncCalibreList(col_read=None, col_toread=None, userid=None):
""" Get the lazylibrarian bookid for each read/toread calibre book so we can map our id to theirs,
and sync current/supplied user's read/toread or supplied read/toread columns to calibre database.
Return message giving totals """
myDB = database.DBConnection()
if not userid:
cookie = cherrypy.request.cookie
if cookie and 'll_uid' in cookie.keys():
userid = cookie['ll_uid'].value
if userid:
res = myDB.match(
'SELECT UserName,ToRead,HaveRead,CalibreRead,CalibreToRead,Perms from users where UserID=?',
(userid, ))
if res:
username = res['UserName']
if not col_read:
col_read = res['CalibreRead']
if not col_toread:
col_toread = res['CalibreToRead']
toreadlist = getList(res['ToRead'])
readlist = getList(res['HaveRead'])
# suppress duplicates (just in case)
toreadlist = list(set(toreadlist))
readlist = list(set(readlist))
else:
return "Error: Unable to get user column settings for %s" % userid
if not userid:
return "Error: Unable to find current userid"
if not col_read and not col_toread:
return "User %s has no calibre columns set" % username
# check user columns exist in calibre and create if not
res = calibredb('custom_columns')
columns = res[0].split('\n')
custom_columns = []
for column in columns:
if column:
custom_columns.append(column.split(' (')[0])
if col_read not in custom_columns:
added = calibredb('add_custom_column', [col_read, col_read, 'bool'])
if "column created" not in added[0]:
return added
if col_toread not in custom_columns:
added = calibredb('add_custom_column',
[col_toread, col_toread, 'bool'])
if "column created" not in added[0]:
return added
nomatch = 0
readcol = ''
toreadcol = ''
map_ctol = {}
map_ltoc = {}
if col_read:
readcol = '*' + col_read
if col_toread:
toreadcol = '*' + col_toread
calibre_list = calibreList(col_read, col_toread)
if not isinstance(calibre_list, list):
# got an error message from calibredb
return '"%s"' % calibre_list
for item in calibre_list:
if toreadcol and toreadcol in item or readcol and readcol in item:
authorname, authorid, added = addAuthorNameToDB(item['authors'],
refresh=False,
addbooks=False)
if authorname:
if authorname != item['authors']:
logger.debug(
"Changed authorname for [%s] from [%s] to [%s]" %
(item['title'], item['authors'], authorname))
item['authors'] = authorname
bookid = find_book_in_db(authorname, item['title'])
if not bookid:
searchterm = "%s <ll> %s" % (item['title'], authorname)
results = search_for(unaccented(searchterm))
if results:
result = results[0]
if result['author_fuzz'] > lazylibrarian.CONFIG['MATCH_RATIO'] \
and result['book_fuzz'] > lazylibrarian.CONFIG['MATCH_RATIO']:
logger.debug(
"Found (%s%% %s%%) %s: %s" %
(result['author_fuzz'], result['book_fuzz'],
result['authorname'], result['bookname']))
bookid = result['bookid']
import_book(bookid)
if bookid:
# NOTE: calibre bookid is always an integer, lazylibrarian bookid is a string
# (goodreads could be used as an int, but googlebooks can't as it's alphanumeric)
# so convert all dict items to strings for ease of matching.
map_ctol[str(item['id'])] = str(bookid)
map_ltoc[str(bookid)] = str(item['id'])
else:
logger.warn(
'Calibre Book [%s] by [%s] is not in lazylibrarian database'
% (item['title'], authorname))
nomatch += 1
else:
logger.warn(
'Calibre Author [%s] not matched in lazylibrarian database'
% (item['authors']))
nomatch += 1
# Now check current users lazylibrarian read/toread against the calibre library, warn about missing ones
# which might be books calibre doesn't have, or might be minor differences in author or title
for idlist in [("Read", readlist), ("To_Read", toreadlist)]:
booklist = idlist[1]
for bookid in booklist:
cmd = "SELECT AuthorID,BookName from books where BookID=?"
book = myDB.match(cmd, (bookid, ))
if not book:
logger.error('Error finding bookid %s' % bookid)
else:
cmd = "SELECT AuthorName from authors where AuthorID=?"
author = myDB.match(cmd, (book['AuthorID'], ))
if not author:
logger.error('Error finding authorid %s' %
book['AuthorID'])
else:
match = False
for item in calibre_list:
if item['authors'] == author['AuthorName'] and item[
'title'] == book['BookName']:
logger.debug("Exact match for %s [%s]" %
(idlist[0], book['BookName']))
map_ctol[str(item['id'])] = str(bookid)
map_ltoc[str(bookid)] = str(item['id'])
match = True
break
if not match:
high = 0
highname = ''
highid = ''
for item in calibre_list:
if item['authors'] == author['AuthorName']:
n = fuzz.token_sort_ratio(
item['title'], book['BookName'])
if n > high:
high = n
highname = item['title']
highid = item['id']
if high > 95:
logger.debug(
"Found ratio match %s%% [%s] for %s [%s]" %
(high, highname, idlist[0], book['BookName']))
map_ctol[str(highid)] = str(bookid)
map_ltoc[str(bookid)] = str(highid)
match = True
if not match:
logger.warn(
"No match for %s %s by %s in calibre database, closest match %s%% [%s]"
% (idlist[0], book['BookName'],
author['AuthorName'], high, highname))
nomatch += 1
logger.debug("BookID mapping complete, %s match %s, nomatch %s" %
(username, len(map_ctol), nomatch))
# now sync the lists
if userid:
last_read = []
last_toread = []
calibre_read = []
calibre_toread = []
cmd = 'select SyncList from sync where UserID=? and Label=?'
res = myDB.match(cmd, (userid, col_read))
if res:
last_read = getList(res['SyncList'])
res = myDB.match(cmd, (userid, col_toread))
if res:
last_toread = getList(res['SyncList'])
for item in calibre_list:
if toreadcol and toreadcol in item and item[
toreadcol]: # only if True
if str(item['id']) in map_ctol:
calibre_toread.append(map_ctol[str(item['id'])])
else:
logger.warn(
"Calibre to_read book %s:%s has no lazylibrarian bookid"
% (item['authors'], item['title']))
if readcol and readcol in item and item[readcol]: # only if True
if str(item['id']) in map_ctol:
calibre_read.append(map_ctol[str(item['id'])])
else:
logger.warn(
"Calibre read book %s:%s has no lazylibrarian bookid" %
(item['authors'], item['title']))
logger.debug("Found %s calibre read, %s calibre toread" %
(len(calibre_read), len(calibre_toread)))
logger.debug("Found %s lazylib read, %s lazylib toread" %
(len(readlist), len(toreadlist)))
added_to_ll_toread = list(set(toreadlist) - set(last_toread))
removed_from_ll_toread = list(set(last_toread) - set(toreadlist))
added_to_ll_read = list(set(readlist) - set(last_read))
removed_from_ll_read = list(set(last_read) - set(readlist))
logger.debug("lazylibrarian changes to copy to calibre: %s %s %s %s" %
(len(added_to_ll_toread), len(removed_from_ll_toread),
len(added_to_ll_read), len(removed_from_ll_read)))
added_to_calibre_toread = list(set(calibre_toread) - set(last_toread))
removed_from_calibre_toread = list(
set(last_toread) - set(calibre_toread))
added_to_calibre_read = list(set(calibre_read) - set(last_read))
removed_from_calibre_read = list(set(last_read) - set(calibre_read))
logger.debug(
"calibre changes to copy to lazylibrarian: %s %s %s %s" %
(len(added_to_calibre_toread), len(removed_from_calibre_toread),
len(added_to_calibre_read), len(removed_from_calibre_read)))
calibre_changes = 0
for item in added_to_calibre_read:
if item not in readlist:
readlist.append(item)
logger.debug("Lazylibrarian marked %s as read" % item)
calibre_changes += 1
for item in added_to_calibre_toread:
if item not in toreadlist:
toreadlist.append(item)
logger.debug("Lazylibrarian marked %s as to_read" % item)
calibre_changes += 1
for item in removed_from_calibre_read:
if item in readlist:
readlist.remove(item)
logger.debug("Lazylibrarian removed %s from read" % item)
calibre_changes += 1
for item in removed_from_calibre_toread:
if item in toreadlist:
toreadlist.remove(item)
logger.debug("Lazylibrarian removed %s from to_read" % item)
calibre_changes += 1
if calibre_changes:
myDB.action('UPDATE users SET ToRead=?,HaveRead=? WHERE UserID=?',
(', '.join(toreadlist), ', '.join(readlist), userid))
ll_changes = 0
for item in added_to_ll_toread:
if item in map_ltoc:
res, err, rc = calibredb('set_custom',
[col_toread, map_ltoc[item], 'true'],
[])
if rc:
msg = "calibredb set_custom error: "
if err:
logger.error(msg + err)
elif res:
logger.error(msg + res)
else:
logger.error(msg + str(rc))
else:
ll_changes += 1
else:
logger.warn("Unable to set calibre %s true for %s" %
(col_toread, item))
for item in removed_from_ll_toread:
if item in map_ltoc:
res, err, rc = calibredb('set_custom',
[col_toread, map_ltoc[item], ''], [])
if rc:
msg = "calibredb set_custom error: "
if err:
logger.error(msg + err)
elif res:
logger.error(msg + res)
else:
logger.error(msg + str(rc))
else:
ll_changes += 1
else:
logger.warn("Unable to clear calibre %s for %s" %
(col_toread, item))
for item in added_to_ll_read:
if item in map_ltoc:
res, err, rc = calibredb('set_custom',
[col_read, map_ltoc[item], 'true'],
[])
if rc:
msg = "calibredb set_custom error: "
if err:
logger.error(msg + err)
elif res:
logger.error(msg + res)
else:
logger.error(msg + str(rc))
else:
ll_changes += 1
else:
logger.warn("Unable to set calibre %s true for %s" %
(col_read, item))
for item in removed_from_ll_read:
if item in map_ltoc:
res, err, rc = calibredb('set_custom',
[col_read, map_ltoc[item], ''], [])
if rc:
msg = "calibredb set_custom error: "
if err:
logger.error(msg + err)
elif res:
logger.error(msg + res)
else:
logger.error(msg + str(rc))
else:
ll_changes += 1
else:
logger.warn("Unable to clear calibre %s for %s" %
(col_read, item))
# store current sync list as comparison for next sync
controlValueDict = {"UserID": userid, "Label": col_read}
newValueDict = {
"Date": str(time.time()),
"Synclist": ', '.join(readlist)
}
myDB.upsert("sync", newValueDict, controlValueDict)
controlValueDict = {"UserID": userid, "Label": col_toread}
newValueDict = {
"Date": str(time.time()),
"Synclist": ', '.join(toreadlist)
}
myDB.upsert("sync", newValueDict, controlValueDict)
msg = "%s sync updated: %s calibre, %s lazylibrarian" % (
username, ll_changes, calibre_changes)
return msg
def processCSV(search_dir=None):
""" Find a csv file in the search_dir and process all the books in it,
adding authors to the database if not found, and marking the books as "Wanted" """
if not search_dir:
logger.warn("Alternate Directory must not be empty")
return False
csvFile = csv_file(search_dir)
headers = None
content = {}
if not csvFile:
logger.warn("No CSV file found in %s" % search_dir)
else:
logger.debug('Reading file %s' % csvFile)
reader=csv.reader(open(csvFile))
for row in reader:
if reader.line_num == 1:
# If we are on the first line, create the headers list from the first row
# by taking a slice from item 1 as we don't need the very first header.
headers = row[1:]
else:
# Otherwise, the key in the content dictionary is the first item in the
# row and we can create the sub-dictionary by using the zip() function.
content[row[0]] = dict(zip(headers, row[1:]))
# We can now get to the content by using the resulting dictionary, so to see
# the list of lines, we can do:
#print content.keys() # to get a list of bookIDs
# To see the list of fields available for each book
#print headers
if 'Author' not in headers or 'Title' not in headers:
logger.warn('Invalid CSV file found %s' % csvFile)
return
myDB = database.DBConnection()
bookcount = 0
authcount = 0
skipcount = 0
logger.debug("CSV: Found %s entries in csv file" % len(content.keys()))
for bookid in content.keys():
authorname = content[bookid]['Author']
authmatch = myDB.action('SELECT * FROM authors where AuthorName="%s"' % (authorname)).fetchone()
if authmatch:
logger.debug("CSV: Author %s found in database" % (authorname))
else:
logger.debug("CSV: Author %s not found, adding to database" % (authorname))
importer.addAuthorToDB(authorname)
authcount = authcount + 1
bookmatch = 0
isbn10=""
isbn13=""
bookname = content[bookid]['Title']
if 'ISBN' in headers:
isbn10 = content[bookid]['ISBN']
if 'ISBN13' in headers:
isbn13 = content[bookid]['ISBN13']
# try to find book in our database using isbn, or if that fails, fuzzy name matching
if formatter.is_valid_isbn(isbn10):
bookmatch = myDB.action('SELECT * FROM books where Bookisbn=%s' % (isbn10)).fetchone()
if not bookmatch:
if formatter.is_valid_isbn(isbn13):
bookmatch = myDB.action('SELECT * FROM books where BookIsbn=%s' % (isbn13)).fetchone()
if not bookmatch:
bookid = librarysync.find_book_in_db(myDB, authorname, bookname)
if bookid:
bookmatch = myDB.action('SELECT * FROM books where BookID="%s"' % (bookid)).fetchone()
if bookmatch:
authorname = bookmatch['AuthorName']
bookname = bookmatch['BookName']
bookid = bookmatch['BookID']
bookstatus = bookmatch['Status']
if bookstatus == 'Open' or bookstatus == 'Wanted' or bookstatus == 'Have':
logger.info('Found book %s by %s, already marked as "%s"' % (bookname, authorname, bookstatus))
else: # skipped/ignored
logger.info('Found book %s by %s, marking as "Wanted"' % (bookname, authorname))
controlValueDict = {"BookID": bookid}
newValueDict = {"Status": "Wanted"}
myDB.upsert("books", newValueDict, controlValueDict)
bookcount = bookcount + 1
else:
logger.warn("Skipping book %s by %s, not found in database" % (bookname, authorname))
skipcount = skipcount + 1
logger.info("Added %i new authors, marked %i books as 'Wanted', %i books not found" % (authcount, bookcount, skipcount))
def processAlternate(source_dir=None):
# import a book from an alternate directory
if not source_dir or os.path.isdir(source_dir) is False:
logger.warn('Alternate directory not found')
return
if source_dir == lazylibrarian.DESTINATION_DIR:
logger.warn('Alternate directory must not be the same as destination')
return
logger.debug('Processing alternate directory %s' % source_dir)
# first, recursively process any books in subdirectories
for fname in os.listdir(source_dir):
subdir = os.path.join(source_dir, fname)
if os.path.isdir(subdir):
processAlternate(subdir)
# only import one book from each alternate (sub)directory, this is because
# the importer may delete the directory after importing a book,
# depending on lazylibrarian.DESTINATION_COPY setting
# also if multiple books in a folder and only a "metadata.opf"
# which book is it for?
new_book = book_file(source_dir, booktype='book')
if new_book:
metadata = {}
# see if there is a metadata file in this folder with the info we need
# try book_name.opf first, or fall back to any filename.opf
metafile = os.path.splitext(new_book)[0] + '.opf'
if not os.path.isfile(metafile):
metafile = librarysync.opf_file(source_dir)
if os.path.isfile(metafile):
try:
metadata = librarysync.get_book_info(metafile)
except:
logger.debug('Failed to read metadata from %s' % metafile)
else:
logger.debug('No metadata file found for %s' % new_book)
if not 'title' in metadata and 'creator' in metadata:
# try to get metadata from the book file
try:
metadata = librarysync.get_book_info(new_book)
except:
logger.debug('No metadata found in %s' % new_book)
if 'title' in metadata and 'creator' in metadata:
authorname = metadata['creator']
bookname = metadata['title']
myDB = database.DBConnection()
authmatch = myDB.action('SELECT * FROM authors where AuthorName="%s"' % (authorname)).fetchone()
if authmatch:
logger.debug("ALT: Author %s found in database" % (authorname))
else:
logger.debug("ALT: Author %s not found, adding to database" % (authorname))
importer.addAuthorToDB(authorname)
bookid = librarysync.find_book_in_db(myDB, authorname, bookname)
if bookid:
import_book(source_dir, bookid)
else:
logger.warn("Book %s by %s not found in database" % (bookname, authorname))
else:
logger.warn('Book %s has no metadata, unable to import' % new_book)
else:
logger.warn("No book file found in %s" % source_dir)
def syncCalibreList(col_read=None, col_toread=None, userid=None):
""" Get the lazylibrarian bookid for each read/toread calibre book so we can map our id to theirs,
and sync current/supplied user's read/toread or supplied read/toread columns to calibre database.
Return message giving totals """
myDB = database.DBConnection()
username = ''
readlist = []
toreadlist = []
if not userid:
cookie = cherrypy.request.cookie
if cookie and 'll_uid' in list(cookie.keys()):
userid = cookie['ll_uid'].value
if userid:
res = myDB.match('SELECT UserName,ToRead,HaveRead,CalibreRead,CalibreToRead,Perms from users where UserID=?',
(userid,))
if res:
username = res['UserName']
if not col_read:
col_read = res['CalibreRead']
if not col_toread:
col_toread = res['CalibreToRead']
toreadlist = getList(res['ToRead'])
readlist = getList(res['HaveRead'])
# suppress duplicates (just in case)
toreadlist = list(set(toreadlist))
readlist = list(set(readlist))
else:
return "Error: Unable to get user column settings for %s" % userid
if not userid:
return "Error: Unable to find current userid"
if not col_read and not col_toread:
return "User %s has no calibre columns set" % username
# check user columns exist in calibre and create if not
res = calibredb('custom_columns')
columns = res[0].split('\n')
custom_columns = []
for column in columns:
if column:
custom_columns.append(column.split(' (')[0])
if col_read not in custom_columns:
added = calibredb('add_custom_column', [col_read, col_read, 'bool'])
if "column created" not in added[0]:
return added
if col_toread not in custom_columns:
added = calibredb('add_custom_column', [col_toread, col_toread, 'bool'])
if "column created" not in added[0]:
return added
nomatch = 0
readcol = ''
toreadcol = ''
map_ctol = {}
map_ltoc = {}
if col_read:
readcol = '*' + col_read
if col_toread:
toreadcol = '*' + col_toread
calibre_list = calibreList(col_read, col_toread)
if not isinstance(calibre_list, list):
# got an error message from calibredb
return '"%s"' % calibre_list
for item in calibre_list:
if toreadcol and toreadcol in item or readcol and readcol in item:
authorname, authorid, added = addAuthorNameToDB(item['authors'], refresh=False, addbooks=False)
if authorname:
if authorname != item['authors']:
logger.debug("Changed authorname for [%s] from [%s] to [%s]" %
(item['title'], item['authors'], authorname))
item['authors'] = authorname
bookid, mtype = find_book_in_db(authorname, item['title'], ignored=False, library='eBook')
if bookid and mtype == "Ignored":
logger.warn("Book %s by %s is marked Ignored in database, importing anyway" %
(item['title'], authorname))
if not bookid:
searchterm = "%s <ll> %s" % (item['title'], authorname)
results = search_for(unaccented(searchterm))
if results:
result = results[0]
if result['author_fuzz'] > lazylibrarian.CONFIG['MATCH_RATIO'] \
and result['book_fuzz'] > lazylibrarian.CONFIG['MATCH_RATIO']:
logger.debug("Found (%s%% %s%%) %s: %s" % (result['author_fuzz'], result['book_fuzz'],
result['authorname'], result['bookname']))
bookid = result['bookid']
import_book(bookid)
if bookid:
# NOTE: calibre bookid is always an integer, lazylibrarian bookid is a string
# (goodreads could be used as an int, but googlebooks can't as it's alphanumeric)
# so convert all dict items to strings for ease of matching.
map_ctol[str(item['id'])] = str(bookid)
map_ltoc[str(bookid)] = str(item['id'])
else:
logger.warn('Calibre Book [%s] by [%s] is not in lazylibrarian database' %
(item['title'], authorname))
nomatch += 1
else:
logger.warn('Calibre Author [%s] not matched in lazylibrarian database' % (item['authors']))
nomatch += 1
# Now check current users lazylibrarian read/toread against the calibre library, warn about missing ones
# which might be books calibre doesn't have, or might be minor differences in author or title
for idlist in [("Read", readlist), ("To_Read", toreadlist)]:
booklist = idlist[1]
for bookid in booklist:
cmd = "SELECT AuthorID,BookName from books where BookID=?"
book = myDB.match(cmd, (bookid,))
if not book:
logger.error('Error finding bookid %s' % bookid)
else:
cmd = "SELECT AuthorName from authors where AuthorID=?"
author = myDB.match(cmd, (book['AuthorID'],))
if not author:
logger.error('Error finding authorid %s' % book['AuthorID'])
else:
match = False
high = 0
highname = ''
for item in calibre_list:
if item['authors'] == author['AuthorName'] and item['title'] == book['BookName']:
logger.debug("Exact match for %s [%s]" % (idlist[0], book['BookName']))
map_ctol[str(item['id'])] = str(bookid)
map_ltoc[str(bookid)] = str(item['id'])
match = True
break
if not match:
highid = ''
for item in calibre_list:
if item['authors'] == author['AuthorName']:
n = fuzz.token_sort_ratio(item['title'], book['BookName'])
if n > high:
high = n
highname = item['title']
highid = item['id']
if high > 95:
logger.debug("Found ratio match %s%% [%s] for %s [%s]" %
(high, highname, idlist[0], book['BookName']))
map_ctol[str(highid)] = str(bookid)
map_ltoc[str(bookid)] = str(highid)
match = True
if not match:
logger.warn("No match for %s %s by %s in calibre database, closest match %s%% [%s]" %
(idlist[0], book['BookName'], author['AuthorName'], high, highname))
nomatch += 1
logger.debug("BookID mapping complete, %s match %s, nomatch %s" % (username, len(map_ctol), nomatch))
# now sync the lists
if not userid:
msg = "No userid found"
else:
last_read = []
last_toread = []
calibre_read = []
calibre_toread = []
cmd = 'select SyncList from sync where UserID=? and Label=?'
res = myDB.match(cmd, (userid, col_read))
if res:
last_read = getList(res['SyncList'])
res = myDB.match(cmd, (userid, col_toread))
if res:
last_toread = getList(res['SyncList'])
for item in calibre_list:
if toreadcol and toreadcol in item and item[toreadcol]: # only if True
if str(item['id']) in map_ctol:
calibre_toread.append(map_ctol[str(item['id'])])
else:
logger.warn("Calibre to_read book %s:%s has no lazylibrarian bookid" %
(item['authors'], item['title']))
if readcol and readcol in item and item[readcol]: # only if True
if str(item['id']) in map_ctol:
calibre_read.append(map_ctol[str(item['id'])])
else:
logger.warn("Calibre read book %s:%s has no lazylibrarian bookid" %
(item['authors'], item['title']))
logger.debug("Found %s calibre read, %s calibre toread" % (len(calibre_read), len(calibre_toread)))
logger.debug("Found %s lazylib read, %s lazylib toread" % (len(readlist), len(toreadlist)))
added_to_ll_toread = list(set(toreadlist) - set(last_toread))
removed_from_ll_toread = list(set(last_toread) - set(toreadlist))
added_to_ll_read = list(set(readlist) - set(last_read))
removed_from_ll_read = list(set(last_read) - set(readlist))
logger.debug("lazylibrarian changes to copy to calibre: %s %s %s %s" % (len(added_to_ll_toread),
len(removed_from_ll_toread), len(added_to_ll_read), len(removed_from_ll_read)))
added_to_calibre_toread = list(set(calibre_toread) - set(last_toread))
removed_from_calibre_toread = list(set(last_toread) - set(calibre_toread))
added_to_calibre_read = list(set(calibre_read) - set(last_read))
removed_from_calibre_read = list(set(last_read) - set(calibre_read))
logger.debug("calibre changes to copy to lazylibrarian: %s %s %s %s" % (len(added_to_calibre_toread),
len(removed_from_calibre_toread), len(added_to_calibre_read), len(removed_from_calibre_read)))
calibre_changes = 0
for item in added_to_calibre_read:
if item not in readlist:
readlist.append(item)
logger.debug("Lazylibrarian marked %s as read" % item)
calibre_changes += 1
for item in added_to_calibre_toread:
if item not in toreadlist:
toreadlist.append(item)
logger.debug("Lazylibrarian marked %s as to_read" % item)
calibre_changes += 1
for item in removed_from_calibre_read:
if item in readlist:
readlist.remove(item)
logger.debug("Lazylibrarian removed %s from read" % item)
calibre_changes += 1
for item in removed_from_calibre_toread:
if item in toreadlist:
toreadlist.remove(item)
logger.debug("Lazylibrarian removed %s from to_read" % item)
calibre_changes += 1
if calibre_changes:
myDB.action('UPDATE users SET ToRead=?,HaveRead=? WHERE UserID=?',
(', '.join(toreadlist), ', '.join(readlist), userid))
ll_changes = 0
for item in added_to_ll_toread:
if item in map_ltoc:
res, err, rc = calibredb('set_custom', [col_toread, map_ltoc[item], 'true'], [])
if rc:
msg = "calibredb set_custom error: "
if err:
logger.error(msg + err)
elif res:
logger.error(msg + res)
else:
logger.error(msg + str(rc))
else:
ll_changes += 1
else:
logger.warn("Unable to set calibre %s true for %s" % (col_toread, item))
for item in removed_from_ll_toread:
if item in map_ltoc:
res, err, rc = calibredb('set_custom', [col_toread, map_ltoc[item], ''], [])
if rc:
msg = "calibredb set_custom error: "
if err:
logger.error(msg + err)
elif res:
logger.error(msg + res)
else:
logger.error(msg + str(rc))
else:
ll_changes += 1
else:
logger.warn("Unable to clear calibre %s for %s" % (col_toread, item))
for item in added_to_ll_read:
if item in map_ltoc:
res, err, rc = calibredb('set_custom', [col_read, map_ltoc[item], 'true'], [])
if rc:
msg = "calibredb set_custom error: "
if err:
logger.error(msg + err)
elif res:
logger.error(msg + res)
else:
logger.error(msg + str(rc))
else:
ll_changes += 1
else:
logger.warn("Unable to set calibre %s true for %s" % (col_read, item))
for item in removed_from_ll_read:
if item in map_ltoc:
res, err, rc = calibredb('set_custom', [col_read, map_ltoc[item], ''], [])
if rc:
msg = "calibredb set_custom error: "
if err:
logger.error(msg + err)
elif res:
logger.error(msg + res)
else:
logger.error(msg + str(rc))
else:
ll_changes += 1
else:
logger.warn("Unable to clear calibre %s for %s" % (col_read, item))
# store current sync list as comparison for next sync
controlValueDict = {"UserID": userid, "Label": col_read}
newValueDict = {"Date": str(time.time()), "Synclist": ', '.join(readlist)}
myDB.upsert("sync", newValueDict, controlValueDict)
controlValueDict = {"UserID": userid, "Label": col_toread}
newValueDict = {"Date": str(time.time()), "Synclist": ', '.join(toreadlist)}
myDB.upsert("sync", newValueDict, controlValueDict)
msg = "%s sync updated: %s calibre, %s lazylibrarian" % (username, ll_changes, calibre_changes)
return msg
def processCSV(search_dir=None):
""" Find a csv file in the search_dir and process all the books in it,
adding authors to the database if not found, and marking the books as "Wanted" """
if not search_dir or os.path.isdir(search_dir) is False:
logger.warn(u"Alternate Directory must not be empty")
return False
csvFile = csv_file(search_dir)
headers = None
content = {}
if not csvFile:
logger.warn(u"No CSV file found in %s" % search_dir)
else:
logger.debug(u'Reading file %s' % csvFile)
reader = csv.reader(open(csvFile))
for row in reader:
if reader.line_num == 1:
# If we are on the first line, create the headers list from the first row
# by taking a slice from item 1 as we don't need the very first header.
headers = row[1:]
else:
# Otherwise, the key in the content dictionary is the first item in the
# row and we can create the sub-dictionary by using the zip() function.
content[row[0]] = dict(zip(headers, row[1:]))
# We can now get to the content by using the resulting dictionary, so to see
# the list of lines, we can do:
# print content.keys() # to get a list of bookIDs
# To see the list of fields available for each book
# print headers
if 'Author' not in headers or 'Title' not in headers:
logger.warn(u'Invalid CSV file found %s' % csvFile)
return
myDB = database.DBConnection()
bookcount = 0
authcount = 0
skipcount = 0
logger.debug(u"CSV: Found %s entries in csv file" % len(content.keys()))
for bookid in content.keys():
authorname = formatter.latinToAscii(content[bookid]['Author'])
authmatch = myDB.action('SELECT * FROM authors where AuthorName="%s"' % (authorname)).fetchone()
if authmatch:
logger.debug(u"CSV: Author %s found in database" % (authorname))
else:
logger.debug(u"CSV: Author %s not found, adding to database" % (authorname))
importer.addAuthorToDB(authorname)
authcount = authcount + 1
bookmatch = 0
isbn10 = ""
isbn13 = ""
bookname = formatter.latinToAscii(content[bookid]['Title'])
if 'ISBN' in headers:
isbn10 = content[bookid]['ISBN']
if 'ISBN13' in headers:
isbn13 = content[bookid]['ISBN13']
# try to find book in our database using isbn, or if that fails, name matching
if formatter.is_valid_isbn(isbn10):
bookmatch = myDB.action('SELECT * FROM books where Bookisbn=%s' % (isbn10)).fetchone()
if not bookmatch:
if formatter.is_valid_isbn(isbn13):
bookmatch = myDB.action('SELECT * FROM books where BookIsbn=%s' % (isbn13)).fetchone()
if not bookmatch:
bookid = librarysync.find_book_in_db(myDB, authorname, bookname)
if bookid:
bookmatch = myDB.action('SELECT * FROM books where BookID="%s"' % (bookid)).fetchone()
if bookmatch:
authorname = bookmatch['AuthorName']
bookname = bookmatch['BookName']
bookid = bookmatch['BookID']
bookstatus = bookmatch['Status']
if bookstatus == 'Open' or bookstatus == 'Wanted' or bookstatus == 'Have':
logger.info(u'Found book %s by %s, already marked as "%s"' % (bookname, authorname, bookstatus))
else: # skipped/ignored
logger.info(u'Found book %s by %s, marking as "Wanted"' % (bookname, authorname))
controlValueDict = {"BookID": bookid}
newValueDict = {"Status": "Wanted"}
myDB.upsert("books", newValueDict, controlValueDict)
bookcount = bookcount + 1
else:
logger.warn(u"Skipping book %s by %s, not found in database" % (bookname, authorname))
skipcount = skipcount + 1
logger.info(u"Added %i new authors, marked %i books as 'Wanted', %i books not found" %
(authcount, bookcount, skipcount))
def processAlternate(source_dir=None):
# import a book from an alternate directory
if not source_dir or os.path.isdir(source_dir) is False:
logger.warn('Alternate directory not found')
return
if source_dir == lazylibrarian.DESTINATION_DIR:
logger.warn('Alternate directory must not be the same as destination')
return
logger.debug('Processing alternate directory %s' % source_dir)
# first, recursively process any books in subdirectories
for fname in os.listdir(source_dir):
subdir = os.path.join(source_dir, fname)
if os.path.isdir(subdir):
processAlternate(subdir)
# only import one book from each alternate (sub)directory, this is because
# the importer may delete the directory after importing a book,
# depending on lazylibrarian.DESTINATION_COPY setting
# also if multiple books in a folder and only a "metadata.opf"
# which book is it for?
new_book = book_file(source_dir, booktype='book')
if new_book:
metadata = {}
# see if there is a metadata file in this folder with the info we need
# try book_name.opf first, or fall back to any filename.opf
metafile = os.path.splitext(new_book)[0] + '.opf'
if not os.path.isfile(metafile):
metafile = librarysync.opf_file(source_dir)
if os.path.isfile(metafile):
try:
metadata = librarysync.get_book_info(metafile)
except:
logger.debug('Failed to read metadata from %s' % metafile)
else:
logger.debug('No metadata file found for %s' % new_book)
if not 'title' in metadata and 'creator' in metadata:
# try to get metadata from the book file
try:
metadata = librarysync.get_book_info(new_book)
except:
logger.debug('No metadata found in %s' % new_book)
if 'title' in metadata and 'creator' in metadata:
authorname = metadata['creator']
bookname = metadata['title']
myDB = database.DBConnection()
authmatch = myDB.action('SELECT * FROM authors where AuthorName="%s"' % (authorname)).fetchone()
if authmatch:
logger.debug("ALT: Author %s found in database" % (authorname))
else:
logger.debug("ALT: Author %s not found, adding to database" % (authorname))
importer.addAuthorToDB(authorname)
bookid = librarysync.find_book_in_db(myDB, authorname, bookname)
if bookid:
import_book(source_dir, bookid)
else:
logger.warn("Book %s by %s not found in database" % (bookname, authorname))
else:
logger.warn('Book %s has no metadata, unable to import' % new_book)
else:
logger.warn("No book file found in %s" % source_dir)