def addAuthorToDB(authorname=None, refresh=False):
myDB = database.DBConnection()
GR = GoodReads(authorname)
query = "SELECT * from authors WHERE AuthorName='%s'" % authorname.replace(
"'", "''")
dbauthor = myDB.match(query)
controlValueDict = {"AuthorName": authorname}
if dbauthor is None:
newValueDict = {
"AuthorID": "0: %s" % (authorname),
"Status": "Loading"
}
logger.debug("Now adding new author: %s to database" % authorname)
else:
newValueDict = {"Status": "Loading"}
logger.debug("Now updating author: %s" % authorname)
myDB.upsert("authors", newValueDict, controlValueDict)
author = GR.find_author_id(refresh=refresh)
if author:
authorid = author['authorid']
authorlink = author['authorlink']
authorimg = author['authorimg']
if 'nophoto' in authorimg:
authorimg = getAuthorImage(authorid)
if authorimg and authorimg.startswith('http'):
newimg = cache_cover(authorid, authorimg)
if newimg:
authorimg = newimg
controlValueDict = {"AuthorName": authorname}
newValueDict = {
"AuthorID": authorid,
"AuthorLink": authorlink,
"AuthorImg": authorimg,
"AuthorBorn": author['authorborn'],
"AuthorDeath": author['authordeath'],
"DateAdded": today(),
"Status": "Loading"
}
myDB.upsert("authors", newValueDict, controlValueDict)
else:
logger.warn(u"Nothing found for %s" % authorname)
myDB.action('DELETE from authors WHERE AuthorName="%s"' % authorname)
return
# process books
if lazylibrarian.BOOK_API == "GoogleBooks":
book_api = GoogleBooks()
book_api.get_author_books(authorid, authorname, refresh=refresh)
elif lazylibrarian.BOOK_API == "GoodReads":
GR.get_author_books(authorid, authorname, refresh=refresh)
update_totals(authorid)
logger.debug("[%s] Author update complete" % authorname)
def addAuthorToDB(authorname=None, refresh=False):
myDB = database.DBConnection()
GR = GoodReads(authorname)
query = "SELECT * from authors WHERE AuthorName='%s'" % authorname.replace("'", "''")
dbauthor = myDB.action(query).fetchone()
controlValueDict = {"AuthorName": authorname}
if dbauthor is None:
newValueDict = {
"AuthorID": "0: %s" % (authorname),
"Status": "Loading"
}
logger.debug("Now adding new author: %s to database" % authorname)
else:
newValueDict = {"Status": "Loading"}
logger.debug("Now updating author: %s" % authorname)
myDB.upsert("authors", newValueDict, controlValueDict)
author = GR.find_author_id(refresh=refresh)
if author:
authorid = author['authorid']
authorlink = author['authorlink']
authorimg = author['authorimg']
if 'nophoto' in authorimg:
authorimg = getAuthorImage(authorid)
if authorimg and authorimg.startswith('http'):
newimg = cache_cover(authorid, authorimg)
if newimg:
authorimg = newimg
controlValueDict = {"AuthorName": authorname}
newValueDict = {
"AuthorID": authorid,
"AuthorLink": authorlink,
"AuthorImg": authorimg,
"AuthorBorn": author['authorborn'],
"AuthorDeath": author['authordeath'],
"DateAdded": today(),
"Status": "Loading"
}
myDB.upsert("authors", newValueDict, controlValueDict)
else:
logger.warn(u"Nothing found for %s" % authorname)
myDB.action('DELETE from authors WHERE AuthorName="%s"' % authorname)
return
# process books
if lazylibrarian.BOOK_API == "GoogleBooks":
book_api = GoogleBooks()
book_api.get_author_books(authorid, authorname, refresh=refresh)
elif lazylibrarian.BOOK_API == "GoodReads":
GR.get_author_books(authorid, authorname, refresh=refresh)
update_totals(authorid)
logger.debug("[%s] Author update complete" % authorname)
def getAuthorImage(authorid=None):
# tbm=isch search images
# tbs=ift:jpg jpeg file type
if not authorid:
logger.error("getAuthorImage: No authorid")
return None
cachedir = os.path.join(str(lazylibrarian.PROG_DIR),
'data' + os.sep + 'images' + os.sep + 'cache')
coverfile = os.path.join(cachedir, authorid + '.jpg')
if os.path.isfile(coverfile): # use cached image if there is one
lazylibrarian.CACHE_HIT = int(lazylibrarian.CACHE_HIT) + 1
logger.debug(u"getAuthorImage: Returning Cached response for %s" %
coverfile)
coverlink = 'images/cache/' + authorid + '.jpg'
return coverlink
lazylibrarian.CACHE_MISS = int(lazylibrarian.CACHE_MISS) + 1
myDB = database.DBConnection()
authors = myDB.select(
'select AuthorName from authors where AuthorID = "%s"' % authorid)
if authors:
authorname = safe_unicode(authors[0][0]).encode(
lazylibrarian.SYS_ENCODING)
safeparams = urllib.quote_plus("%s" % authorname)
URL = "https://www.google.com/search?tbm=isch&tbs=ift:jpg&as_q=" + safeparams
result, success = fetchURL(URL)
if success:
try:
img = result.split('url?q=')[1].split('">')[1].split(
'src="')[1].split('"')[0]
except IndexError:
img = None
if img and img.startswith('http'):
coverlink = cache_cover(authorid, img)
if coverlink is not None:
logger.debug("Cached google image for %s" % authorname)
return coverlink
else:
logger.debug("Error getting google image %s, [%s]" %
(img, result))
else:
logger.debug("No image found in google page for %s" %
authorname)
else:
logger.debug("Error getting google page for %s, [%s]" %
(safeparams, result))
else:
logger.debug("No author found for %s" % authorid)
return None
def getAuthorImage(authorid=None):
# tbm=isch search images
# tbs=ift:jpg jpeg file type
if not authorid:
logger.error("getAuthorImage: No authorid")
return None
cachedir = lazylibrarian.CACHEDIR
coverfile = os.path.join(cachedir, authorid + '.jpg')
if os.path.isfile(coverfile): # use cached image if there is one
lazylibrarian.CACHE_HIT = int(lazylibrarian.CACHE_HIT) + 1
logger.debug(u"getAuthorImage: Returning Cached response for %s" % coverfile)
coverlink = 'cache/' + authorid + '.jpg'
return coverlink
lazylibrarian.CACHE_MISS = int(lazylibrarian.CACHE_MISS) + 1
myDB = database.DBConnection()
authors = myDB.select('select AuthorName from authors where AuthorID = "%s"' % authorid)
if authors:
authorname = safe_unicode(authors[0][0]).encode(lazylibrarian.SYS_ENCODING)
safeparams = urllib.quote_plus("%s" % authorname)
URL = "https://www.google.com/search?tbm=isch&tbs=ift:jpg&as_q=" + safeparams
result, success = fetchURL(URL)
if success:
try:
img = result.split('url?q=')[1].split('">')[1].split('src="')[1].split('"')[0]
except IndexError:
img = None
if img and img.startswith('http'):
coverlink = cache_cover(authorid, img)
if coverlink:
logger.debug("Cached google image for %s" % authorname)
return coverlink
else:
logger.debug("Error getting google image %s, [%s]" % (img, result))
else:
logger.debug("No image found in google page for %s" % authorname)
else:
logger.debug("Error getting google page for %s, [%s]" % (safeparams, result))
else:
logger.debug("No author found for %s" % authorid)
return None
def find_book(self, bookid=None, queue=None):
myDB = database.DBConnection()
URL = 'https://www.goodreads.com/book/show/' + bookid + '?' + urllib.urlencode(
self.params)
try:
rootxml, in_cache = get_xml_request(URL)
if rootxml is None:
logger.debug("Error requesting book")
return
except Exception as e:
logger.error("Error finding book: %s" % e)
return
bookLanguage = rootxml.find('./book/language_code').text
bookname = rootxml.find('./book/title').text
if not bookLanguage:
bookLanguage = "Unknown"
#
# PAB user has said they want this book, don't block for bad language, just warn
#
valid_langs = ([
valid_lang.strip()
for valid_lang in lazylibrarian.IMP_PREFLANG.split(',')
])
if bookLanguage not in valid_langs:
logger.debug('Book %s language does not match preference' %
bookname)
if (rootxml.find('./book/publication_year').text is None):
bookdate = "0000"
else:
bookdate = rootxml.find('./book/publication_year').text
try:
bookimg = rootxml.find('./book/img_url').text
if 'assets/nocover' in bookimg:
bookimg = 'images/nocover.png'
except (KeyError, AttributeError):
bookimg = 'images/nocover.png'
authorname = rootxml.find('./book/authors/author/name').text
bookdesc = rootxml.find('./book/description').text
bookisbn = rootxml.find('./book/isbn').text
bookpub = rootxml.find('./book/publisher').text
booklink = rootxml.find('./book/link').text
bookrate = float(rootxml.find('./book/average_rating').text)
bookpages = rootxml.find('.book/num_pages').text
name = authorname
GR = GoodReads(name)
author = GR.find_author_id()
if author:
AuthorID = author['authorid']
booksub = ''
bookname = unaccented(bookname)
if ': ' in bookname:
parts = bookname.split(': ', 1)
bookname = parts[0]
booksub = parts[1]
dic = {':': '', '"': '', '\'': ''}
bookname = replace_all(bookname, dic)
bookname = bookname.strip() # strip whitespace
booksub = replace_all(booksub, dic)
booksub = booksub.strip() # strip whitespace
if booksub:
series, seriesNum = bookSeries(booksub)
else:
series, seriesNum = bookSeries(bookname)
controlValueDict = {"BookID": bookid}
newValueDict = {
"AuthorName": authorname,
"AuthorID": AuthorID,
"AuthorLink": None,
"BookName": bookname,
"BookSub": booksub,
"BookDesc": bookdesc,
"BookIsbn": bookisbn,
"BookPub": bookpub,
"BookGenre": None,
"BookImg": bookimg,
"BookLink": booklink,
"BookRate": bookrate,
"BookPages": bookpages,
"BookDate": bookdate,
"BookLang": bookLanguage,
"Status": "Wanted",
"BookAdded": today(),
"Series": series,
"SeriesNum": seriesNum
}
myDB.upsert("books", newValueDict, controlValueDict)
logger.debug("%s added to the books database" % bookname)
if 'nocover' in bookimg or 'nophoto' in bookimg:
# try to get a cover from librarything
workcover = getBookCover(bookid)
if workcover:
logger.debug(u'Updated cover for %s to %s' %
(bookname, workcover))
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": workcover}
myDB.upsert("books", newValueDict, controlValueDict)
elif bookimg and bookimg.startswith('http'):
link = cache_cover(bookid, bookimg)
if link is not None:
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": link}
myDB.upsert("books", newValueDict, controlValueDict)
if seriesNum == None:
# try to get series info from librarything
series, seriesNum = getWorkSeries(bookid)
if seriesNum:
logger.debug(u'Updated series: %s [%s]' % (series, seriesNum))
controlValueDict = {"BookID": bookid}
newValueDict = {"Series": series, "SeriesNum": seriesNum}
myDB.upsert("books", newValueDict, controlValueDict)
worklink = getWorkPage(bookid)
if worklink:
controlValueDict = {"BookID": bookid}
newValueDict = {"WorkPage": worklink}
myDB.upsert("books", newValueDict, controlValueDict)
def get_author_books(self, authorid=None, authorname=None, refresh=False):
api_hits = 0
gr_lang_hits = 0
lt_lang_hits = 0
gb_lang_change = 0
cache_hits = 0
not_cached = 0
URL = 'http://www.goodreads.com/author/list/' + authorid + '.xml?' + urllib.urlencode(
self.params)
# Artist is loading
myDB = database.DBConnection()
controlValueDict = {"AuthorID": authorid}
newValueDict = {"Status": "Loading"}
myDB.upsert("authors", newValueDict, controlValueDict)
books_dict = []
try:
rootxml, in_cache = get_xml_request(URL, useCache=not refresh)
except Exception as e:
logger.error("Error fetching author books: %s" % e)
return books_dict
if rootxml is None:
logger.debug("Error requesting author books")
return books_dict
if not in_cache:
api_hits = api_hits + 1
resultxml = rootxml.getiterator('book')
valid_langs = ([
valid_lang.strip()
for valid_lang in lazylibrarian.IMP_PREFLANG.split(',')
])
if not len(resultxml):
logger.warn('[%s] No books found for author with ID: %s' %
(authorname, authorid))
else:
logger.debug("[%s] Now processing books with GoodReads API" %
authorname)
resultsCount = 0
removedResults = 0
duplicates = 0
ignored = 0
added_count = 0
updated_count = 0
book_ignore_count = 0
total_count = 0
logger.debug(u"url " + URL)
authorNameResult = rootxml.find('./author/name').text
logger.debug(u"author name " + authorNameResult)
loopCount = 1
while resultxml is not None:
for book in resultxml:
total_count = total_count + 1
if (book.find('publication_year').text is None):
pubyear = "0000"
else:
pubyear = book.find('publication_year').text
try:
bookimg = book.find('image_url').text
if ('nocover' in bookimg):
bookimg = 'images/nocover.png'
except (KeyError, AttributeError):
bookimg = 'images/nocover.png'
# PAB this next section tries to get the book language using the isbn13 to look it up. If no isbn13 we skip the
# book entirely, rather than including it with an "Unknown" language. Changed this so we can still include the book
# with language set to "Unknown". There is a setting in config.ini to allow or skip books with "Unknown" language
# if you really don't want to include them.
# Not all GR books have isbn13 filled in, but all have a GR bookid, which we've already got, so use that.
# Also, with GR API rules we can only call the API once per second, which slows us down a lot when all we want
# is to get the language. We sleep for one second per book that GR knows about for each author you have in your
# library. The libraryThing API has the same 1 second restriction, and is limited to 1000 hits per day, but has
# fewer books with unknown language. To get around this and speed up the process, see if we already have a book
# in the database with a similar start to the ISBN. The way ISBNs work, digits 3-5 of a 13 char ISBN or digits 0-2
# of a 10 digit ISBN indicate the region/language so if two books have the same 3 digit isbn code, they _should_
# be the same language.
# I ran a simple python script on my library of 1500 books, and these codes were 100% correct on matching book
# languages, no mis-matches. It did result in a small number of books with "unknown" language being wrongly matched
# but most "unknown" were matched to the correct language.
# We could look up ISBNs we already know about in the database, but this only holds books in the languages we want
# to keep, which reduces the number of cache hits, so we create a new database table, holding ALL results including
# the ISBNs for languages we don't want and books we reject.
# The new table is created (if not exists) in init.py so by the time we get here there is an existing table.
# If we haven't an already matching partial ISBN, look up language code from libraryThing
# "http://www.librarything.com/api/thingLang.php?isbn=1234567890"
# If you find a matching language, add it to the database. If "unknown" or "invalid", try GR as maybe GR can
# provide a match.
# If both LT and GR return unknown, add isbn to db as "unknown". No point in repeatedly asking LT for a code
# it's told you it doesn't know.
# As an extra option, if language includes "All" in config.ini, we can skip this whole section and process
# everything much faster by not querying for language at all.
# It does mean we include a lot of unwanted foreign translations in the database, but it's _much_ faster.
bookLanguage = "Unknown"
find_field = "id"
isbn = ""
isbnhead = ""
if "All" not in valid_langs: # do we care about language
if (book.find('isbn').text is not None):
find_field = "isbn"
isbn = book.find('isbn').text
isbnhead = isbn[0:3]
else:
if (book.find('isbn13').text is not None):
find_field = "isbn13"
isbn = book.find('isbn13').text
isbnhead = isbn[3:6]
if (find_field != 'id'): # isbn or isbn13 found
match = myDB.action(
'SELECT lang FROM languages where isbn = "%s"'
% (isbnhead)).fetchone()
if (match):
bookLanguage = match['lang']
cache_hits = cache_hits + 1
logger.debug(
"Found cached language [%s] for %s [%s]" %
(bookLanguage, find_field, isbnhead))
else:
# no match in cache, try searching librarything for a language code using the isbn
# if no language found, librarything return value is "invalid" or "unknown"
# returns plain text, not xml
BOOK_URL = 'http://www.librarything.com/api/thingLang.php?isbn=' + isbn
try:
librarything_wait()
resp = urllib2.urlopen(BOOK_URL,
timeout=30).read()
lt_lang_hits = lt_lang_hits + 1
logger.debug(
"LibraryThing reports language [%s] for %s"
% (resp, isbnhead))
if ('invalid' in resp
or 'Unknown' in resp):
find_field = "id" # reset the field to force search on goodreads
else:
bookLanguage = resp # found a language code
myDB.action(
'insert into languages values ("%s", "%s")'
% (isbnhead, bookLanguage))
logger.debug(u"LT language %s: %s" %
(isbnhead, bookLanguage))
except Exception as e:
logger.error(
"Error finding LT language result for [%s], %s"
% (isbn, e))
find_field = "id" # reset the field to search on goodreads
if (find_field == 'id'):
# [or bookLanguage == "Unknown"] no earlier match, we'll have to search the goodreads api
try:
if (book.find(find_field).text is not None):
BOOK_URL = 'http://www.goodreads.com/book/show?id=' + \
book.find(find_field).text + '&' + urllib.urlencode(self.params)
logger.debug(u"Book URL: " + BOOK_URL)
try:
time_now = int(time.time())
if time_now <= lazylibrarian.LAST_GOODREADS:
time.sleep(1)
BOOK_rootxml, in_cache = get_xml_request(
BOOK_URL)
if BOOK_rootxml is None:
logger.debug(
'Error requesting book language code'
)
bookLanguage = ""
else:
if not in_cache:
# only update last_goodreads if the result wasn't found in the cache
lazylibrarian.LAST_GOODREADS = time_now
bookLanguage = BOOK_rootxml.find(
'./book/language_code').text
except Exception as e:
logger.error(
"Error finding book results: %s" %
e)
if not in_cache:
gr_lang_hits = gr_lang_hits + 1
if not bookLanguage:
bookLanguage = "Unknown"
if (isbnhead != ""):
# GR didn't give an isbn so we can't cache it, just use language for this book
myDB.action(
'insert into languages values ("%s", "%s")'
% (isbnhead, bookLanguage))
logger.debug(
"GoodReads reports language [%s] for %s"
% (bookLanguage, isbnhead))
else:
not_cached = not_cached + 1
logger.debug(u"GR language: " +
bookLanguage)
else:
logger.debug(
"No %s provided for [%s]" %
(find_field, book.find('title').text))
# continue
except Exception as e:
logger.debug(u"An error has occured: %s" % e)
if bookLanguage not in valid_langs:
logger.debug('Skipped a book with language %s' %
bookLanguage)
ignored = ignored + 1
continue
bookname = book.find('title').text
bookid = book.find('id').text
bookdesc = book.find('description').text
bookisbn = book.find('isbn').text
bookpub = book.find('publisher').text
booklink = book.find('link').text
bookrate = float(book.find('average_rating').text)
bookpages = book.find('num_pages').text
bookname = unaccented(bookname)
if ': ' in bookname:
parts = bookname.split(': ', 1)
bookname = parts[0]
booksub = parts[1]
else:
booksub = ''
dic = {':': '', '"': '', '\'': ''}
bookname = replace_all(bookname, dic)
bookname = bookname.strip() # strip whitespace
booksub = replace_all(booksub, dic)
booksub = booksub.strip() # strip whitespace
if booksub:
series, seriesNum = bookSeries(booksub)
else:
series, seriesNum = bookSeries(bookname)
# GoodReads sometimes has multiple bookids for the same book (same author/title, different editions)
# and sometimes uses the same bookid if the book is the same but the title is slightly different
# We use bookid, then reject if another author/title has a different bookid so we just keep one...
find_book_status = myDB.select(
'SELECT * FROM books WHERE BookID = "%s"' % bookid)
if find_book_status:
for resulted in find_book_status:
book_status = resulted['Status']
locked = resulted['Manual']
else:
book_status = lazylibrarian.NEWBOOK_STATUS
locked = False
rejected = False
if re.match('[^\w-]', bookname
): # reject books with bad characters in title
logger.debug(u"removed result [" + bookname +
"] for bad characters")
removedResults = removedResults + 1
rejected = True
if not rejected and not bookname:
logger.debug(
'Rejecting bookid %s for %s, no bookname' %
(bookid, authorNameResult))
removedResults = removedResults + 1
rejected = True
if not rejected:
find_books = myDB.select(
'SELECT * FROM books WHERE BookName = "%s" and AuthorName = "%s"'
% (bookname, authorNameResult))
if find_books:
for find_book in find_books:
if find_book['BookID'] != bookid:
# we have a book with this author/title already
logger.debug(
'Rejecting bookid %s for [%s][%s] already got %s'
% (find_book['BookID'],
authorNameResult, bookname, bookid))
duplicates = duplicates + 1
rejected = True
break
if not rejected:
find_books = myDB.select(
'SELECT * FROM books WHERE BookID = "%s"' % bookid)
if find_books:
# we have a book with this bookid already
logger.debug(
'Rejecting bookid %s for [%s][%s] already got this bookid in database'
% (bookid, authorNameResult, bookname))
duplicates = duplicates + 1
rejected = True
break
if not rejected:
if book_status != "Ignored":
if not locked:
controlValueDict = {"BookID": bookid}
newValueDict = {
"AuthorName": authorNameResult,
"AuthorID": authorid,
"AuthorLink": None,
"BookName": bookname,
"BookSub": booksub,
"BookDesc": bookdesc,
"BookIsbn": bookisbn,
"BookPub": bookpub,
"BookGenre": None,
"BookImg": bookimg,
"BookLink": booklink,
"BookRate": bookrate,
"BookPages": bookpages,
"BookDate": pubyear,
"BookLang": bookLanguage,
"Status": book_status,
"BookAdded": today(),
"Series": series,
"SeriesNum": seriesNum
}
resultsCount = resultsCount + 1
myDB.upsert("books", newValueDict,
controlValueDict)
logger.debug(u"Book found: " +
book.find('title').text + " " +
pubyear)
if 'nocover' in bookimg or 'nophoto' in bookimg:
# try to get a cover from librarything
workcover = getBookCover(bookid)
if workcover:
logger.debug(
u'Updated cover for %s to %s' %
(bookname, workcover))
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": workcover}
myDB.upsert("books", newValueDict,
controlValueDict)
elif bookimg and bookimg.startswith('http'):
link = cache_cover(bookid, bookimg)
if link is not None:
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": link}
myDB.upsert("books", newValueDict,
controlValueDict)
if seriesNum == None:
# try to get series info from librarything
series, seriesNum = getWorkSeries(bookid)
if seriesNum:
logger.debug(u'Updated series: %s [%s]' %
(series, seriesNum))
controlValueDict = {"BookID": bookid}
newValueDict = {
"Series": series,
"SeriesNum": seriesNum
}
myDB.upsert("books", newValueDict,
controlValueDict)
worklink = getWorkPage(bookid)
if worklink:
controlValueDict = {"BookID": bookid}
newValueDict = {"WorkPage": worklink}
myDB.upsert("books", newValueDict,
controlValueDict)
if not find_book_status:
logger.debug(u"[%s] Added book: %s" %
(authorname, bookname))
added_count = added_count + 1
else:
logger.debug(u"[%s] Updated book: %s" %
(authorname, bookname))
updated_count = updated_count + 1
else:
book_ignore_count = book_ignore_count + 1
loopCount = loopCount + 1
URL = 'http://www.goodreads.com/author/list/' + authorid + '.xml?' + \
urllib.urlencode(self.params) + '&page=' + str(loopCount)
resultxml = None
try:
rootxml, in_cache = get_xml_request(URL,
useCache=not refresh)
if rootxml is None:
logger.debug('Error requesting next page of results')
else:
resultxml = rootxml.getiterator('book')
if not in_cache:
api_hits = api_hits + 1
except Exception as e:
resultxml = None
logger.error("Error finding next page of results: %s" % e)
if resultxml is not None:
if all(False for book in
resultxml): # returns True if iterator is empty
resultxml = None
lastbook = myDB.action(
'SELECT BookName, BookLink, BookDate from books WHERE AuthorID="%s" \
AND Status != "Ignored" order by BookDate DESC'
% authorid).fetchone()
if lastbook:
lastbookname = lastbook['BookName']
lastbooklink = lastbook['BookLink']
lastbookdate = lastbook['BookDate']
else:
lastbookname = None
lastbooklink = None
lastbookdate = None
controlValueDict = {"AuthorID": authorid}
newValueDict = {
"Status": "Active",
"LastBook": lastbookname,
"LastLink": lastbooklink,
"LastDate": lastbookdate
}
myDB.upsert("authors", newValueDict, controlValueDict)
# This is here because GoodReads sometimes has several entries with the same BookID!
modified_count = added_count + updated_count
logger.debug("Found %s total book%s for author" %
(total_count, plural(total_count)))
logger.debug("Removed %s bad language result%s for author" %
(ignored, plural(ignored)))
logger.debug(
"Removed %s bad character or no-name result%s for author" %
(removedResults, plural(removedResults)))
logger.debug("Removed %s duplicate result%s for author" %
(duplicates, plural(duplicates)))
logger.debug("Ignored %s book%s by author marked as Ignored" %
(book_ignore_count, plural(book_ignore_count)))
logger.debug("Imported/Updated %s book%s for author" %
(modified_count, plural(modified_count)))
myDB.action(
'insert into stats values ("%s", %i, %i, %i, %i, %i, %i, %i, %i, %i)'
%
(authorname, api_hits, gr_lang_hits, lt_lang_hits, gb_lang_change,
cache_hits, ignored, removedResults, not_cached, duplicates))
if refresh:
logger.info(
"[%s] Book processing complete: Added %s book%s / Updated %s book%s"
% (authorname, added_count, plural(added_count), updated_count,
plural(updated_count)))
else:
logger.info(
"[%s] Book processing complete: Added %s book%s to the database"
% (authorname, added_count, plural(added_count)))
return books_dict
def find_book(self, bookid=None, queue=None):
myDB = database.DBConnection()
if not lazylibrarian.GB_API:
logger.warn('No GoogleBooks API key, check config')
URL = 'https://www.googleapis.com/books/v1/volumes/' + \
str(bookid) + "?key=" + lazylibrarian.GB_API
jsonresults, in_cache = get_json_request(URL)
if jsonresults is None:
logger.debug('No results found for %s' % bookname)
return
bookname = jsonresults['volumeInfo']['title']
dic = {':': '', '"': '', '\'': ''}
bookname = replace_all(bookname, dic)
bookname = unaccented(bookname)
bookname = bookname.strip() # strip whitespace
try:
authorname = jsonresults['volumeInfo']['authors'][0]
except KeyError:
logger.debug(
'Book %s does not contain author field, skipping' %
bookname)
return
try:
# warn if language is in ignore list, but user said they wanted
# this book
booklang = jsonresults['volumeInfo']['language']
valid_langs = ([valid_lang.strip()
for valid_lang in lazylibrarian.IMP_PREFLANG.split(',')])
if booklang not in valid_langs:
logger.debug(
'Book %s language does not match preference' %
bookname)
except KeyError:
logger.debug('Book does not have language field')
booklang = "Unknown"
try:
bookpub = jsonresults['volumeInfo']['publisher']
except KeyError:
bookpub = None
series = None
seriesNum = None
try:
booksub = jsonresults['volumeInfo']['subtitle']
try:
series = booksub.split('(')[1].split(' Series ')[0]
except IndexError:
series = None
try:
seriesNum = booksub.split('(')[1].split(' Series ')[1].split(')')[0]
if seriesNum[0] == '#':
seriesNum = seriesNum[1:]
except IndexError:
seriesNum = None
except KeyError:
booksub = None
try:
bookdate = jsonresults['volumeInfo']['publishedDate']
except KeyError:
bookdate = '0000-00-00'
try:
bookimg = jsonresults['volumeInfo']['imageLinks']['thumbnail']
except KeyError:
bookimg = 'images/nocover.png'
try:
bookrate = jsonresults['volumeInfo']['averageRating']
except KeyError:
bookrate = 0
try:
bookpages = jsonresults['volumeInfo']['pageCount']
except KeyError:
bookpages = 0
try:
bookgenre = jsonresults['volumeInfo']['categories'][0]
except KeyError:
bookgenre = None
try:
bookdesc = jsonresults['volumeInfo']['description']
except KeyError:
bookdesc = None
try:
if jsonresults['volumeInfo']['industryIdentifiers'][0]['type'] == 'ISBN_10':
bookisbn = jsonresults['volumeInfo'][
'industryIdentifiers'][0]['identifier']
else:
bookisbn = None
except KeyError:
bookisbn = None
booklink = jsonresults['volumeInfo']['canonicalVolumeLink']
bookrate = float(bookrate)
name = jsonresults['volumeInfo']['authors'][0]
GR = GoodReads(name)
author = GR.find_author_id()
if author:
AuthorID = author['authorid']
controlValueDict = {"BookID": bookid}
newValueDict = {
"AuthorName": authorname,
"AuthorID": AuthorID,
"AuthorLink": "",
"BookName": bookname,
"BookSub": booksub,
"BookDesc": bookdesc,
"BookIsbn": bookisbn,
"BookPub": bookpub,
"BookGenre": bookgenre,
"BookImg": bookimg,
"BookLink": booklink,
"BookRate": bookrate,
"BookPages": bookpages,
"BookDate": bookdate,
"BookLang": booklang,
"Status": "Wanted",
"BookAdded": today(),
"Series": series,
"SeriesNum": seriesNum
}
myDB.upsert("books", newValueDict, controlValueDict)
logger.debug("%s added to the books database" % bookname)
if 'nocover' in bookimg or 'nophoto' in bookimg:
# try to get a cover from librarything
workcover = getBookCover(bookid)
if workcover:
logger.debug(u'Updated cover for %s to %s' % (bookname, workcover))
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": workcover}
myDB.upsert("books", newValueDict, controlValueDict)
elif bookimg and bookimg.startswith('http'):
link = cache_cover(bookid, bookimg)
if link is not None:
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": link}
myDB.upsert("books", newValueDict, controlValueDict)
if seriesNum is None:
# try to get series info from librarything
series, seriesNum = getWorkSeries(bookid)
if seriesNum:
logger.debug(u'Updated series: %s [%s]' % (series, seriesNum))
controlValueDict = {"BookID": bookid}
newValueDict = {
"Series": series,
"SeriesNum": seriesNum
}
myDB.upsert("books", newValueDict, controlValueDict)
worklink = getWorkPage(bookid)
if worklink:
controlValueDict = {"BookID": bookid}
newValueDict = {"WorkPage": worklink}
myDB.upsert("books", newValueDict, controlValueDict)
def get_author_books(self, authorid=None, authorname=None, refresh=False):
logger.debug('[%s] Now processing books with Google Books API' % authorname)
# google doesnt like accents in author names
set_url = self.url + urllib.quote('inauthor:"%s"' % unaccented_str(authorname))
URL = set_url + '&' + urllib.urlencode(self.params)
books_dict = []
api_hits = 0
gr_lang_hits = 0
lt_lang_hits = 0
gb_lang_change = 0
cache_hits = 0
not_cached = 0
# Artist is loading
myDB = database.DBConnection()
controlValueDict = {"AuthorID": authorid}
newValueDict = {"Status": "Loading"}
myDB.upsert("authors", newValueDict, controlValueDict)
try:
startindex = 0
resultcount = 0
removedResults = 0
duplicates = 0
ignored = 0
added_count = 0
updated_count = 0
book_ignore_count = 0
total_count = 0
number_results = 1
valid_langs = ([valid_lang.strip()
for valid_lang in lazylibrarian.IMP_PREFLANG.split(',')])
while startindex < number_results:
self.params['startIndex'] = startindex
URL = set_url + '&' + urllib.urlencode(self.params)
try:
jsonresults, in_cache = get_json_request(URL, useCache=not refresh)
if jsonresults is None:
number_results = 0
else:
if not in_cache:
api_hits = api_hits + 1
number_results = jsonresults['totalItems']
except HTTPError as err:
logger.warn(
'Google Books API Error [%s]: Check your API key or wait a while' %
err.reason)
break
if number_results == 0:
logger.warn('Found no results for %s' % authorname)
break
else:
logger.debug('Found %s result%s for %s' % (number_results, plural(number_results), authorname))
startindex = startindex + 40
for item in jsonresults['items']:
total_count = total_count + 1
# skip if no author, no author is no book.
try:
Author = item['volumeInfo']['authors'][0]
except KeyError:
logger.debug('Skipped a result without authorfield.')
continue
try:
if item['volumeInfo']['industryIdentifiers'][0]['type'] == 'ISBN_10':
bookisbn = item['volumeInfo'][
'industryIdentifiers'][0]['identifier']
else:
bookisbn = ""
except KeyError:
bookisbn = ""
isbnhead = ""
if len(bookisbn) == 10:
isbnhead = bookisbn[0:3]
try:
booklang = item['volumeInfo']['language']
except KeyError:
booklang = "Unknown"
# do we care about language?
if "All" not in valid_langs:
if bookisbn != "":
# seems google lies to us, sometimes tells us books
# are in english when they are not
if booklang == "Unknown" or booklang == "en":
googlelang = booklang
match = myDB.match('SELECT lang FROM languages where isbn = "%s"' %
(isbnhead))
if (match):
booklang = match['lang']
cache_hits = cache_hits + 1
logger.debug(
"Found cached language [%s] for [%s]" %
(booklang, isbnhead))
else:
# no match in cache, try searching librarything for a language code using the isbn
# if no language found, librarything return value is "invalid" or "unknown"
# librarything returns plain text, not xml
BOOK_URL = 'http://www.librarything.com/api/thingLang.php?isbn=' + \
bookisbn
try:
librarything_wait()
resp = urllib2.urlopen(BOOK_URL, timeout=30).read()
lt_lang_hits = lt_lang_hits + 1
logger.debug(
"LibraryThing reports language [%s] for %s" % (resp, isbnhead))
if (resp != 'invalid' and resp != 'unknown'):
booklang = resp # found a language code
myDB.action('insert into languages values ("%s", "%s")' %
(isbnhead, booklang))
logger.debug(u"LT language: " + booklang)
except Exception as e:
booklang = ""
logger.error("Error finding language: %s" % str(e))
if googlelang == "en" and booklang not in ["en-US", "en-GB", "eng"]:
# these are all english, may need to expand
# this list
booknamealt = item['volumeInfo']['title']
logger.debug("%s Google thinks [%s], we think [%s]" %
(booknamealt, googlelang, booklang))
gb_lang_change = gb_lang_change + 1
else:
match = myDB.match('SELECT lang FROM languages where isbn = "%s"' %
(isbnhead))
if (not match):
myDB.action(
'insert into languages values ("%s", "%s")' %
(isbnhead, booklang))
logger.debug(u"GB language: " + booklang)
# skip if language is in ignore list
if booklang not in valid_langs:
booknamealt = item['volumeInfo']['title']
logger.debug(
'Skipped [%s] with language %s' %
(booknamealt, booklang))
ignored = ignored + 1
continue
try:
bookpub = item['volumeInfo']['publisher']
except KeyError:
bookpub = None
try:
booksub = item['volumeInfo']['subtitle']
except KeyError:
booksub = None
if booksub is None:
series = None
seriesNum = None
else:
try:
series = booksub.split('(')[1].split(' Series ')[0]
except IndexError:
series = None
try:
seriesNum = booksub.split('(')[1].split(' Series ')[1].split(')')[0]
if seriesNum[0] == '#':
seriesNum = seriesNum[1:]
except IndexError:
seriesNum = None
try:
bookdate = item['volumeInfo']['publishedDate']
except KeyError:
bookdate = '0000-00-00'
try:
bookimg = item['volumeInfo']['imageLinks']['thumbnail']
except KeyError:
bookimg = 'images/nocover.png'
try:
bookrate = item['volumeInfo']['averageRating']
except KeyError:
bookrate = 0
try:
bookpages = item['volumeInfo']['pageCount']
except KeyError:
bookpages = 0
try:
bookgenre = item['volumeInfo']['categories'][0]
except KeyError:
bookgenre = None
try:
bookdesc = item['volumeInfo']['description']
except KeyError:
bookdesc = None
bookname = item['volumeInfo']['title']
bookname = unaccented(bookname)
dic = {':': '', '"': '', '\'': ''}
bookname = replace_all(bookname, dic)
bookname = bookname.strip() # strip whitespace
booklink = item['volumeInfo']['canonicalVolumeLink']
bookrate = float(bookrate)
bookid = item['id']
# GoodReads sometimes has multiple bookids for the same book (same author/title, different editions)
# and sometimes uses the same bookid if the book is the same but the title is slightly different
#
# Not sure if googlebooks does too, but we only want one...
find_book_status = myDB.select('SELECT * FROM books WHERE BookID = "%s"' % bookid)
if find_book_status:
for resulted in find_book_status:
book_status = resulted['Status']
locked = resulted['Manual']
else:
book_status = lazylibrarian.NEWBOOK_STATUS
locked = False
rejected = False
if re.match('[^\w-]', bookname): # remove books with bad characters in title
logger.debug("[%s] removed book for bad characters" % bookname)
removedResults = removedResults + 1
rejected = True
if not rejected and not bookname:
logger.debug('Rejecting bookid %s for %s, no bookname' %
(bookid, authorname))
removedResults = removedResults + 1
rejected = True
if not rejected:
find_books = myDB.select('SELECT * FROM books WHERE BookName = "%s" and AuthorName = "%s"' %
(bookname.replace('"', '""'), authorname.replace('"', '""')))
if find_books:
for find_book in find_books:
if find_book['BookID'] != bookid:
# we have a book with this author/title already
logger.debug('Rejecting bookid %s for [%s][%s] already got %s' %
(find_book['BookID'], authorname, bookname, bookid))
rejected = True
duplicates = duplicates + 1
if not rejected:
find_books = myDB.select('SELECT * FROM books WHERE BookID = "%s"' % bookid)
if find_books:
# we have a book with this bookid already
logger.debug('Rejecting bookid %s for [%s][%s] already got this bookid in database' %
(bookid, authorname, bookname))
duplicates = duplicates + 1
rejected = True
if not rejected:
if book_status != "Ignored" and not locked:
controlValueDict = {"BookID": bookid}
newValueDict = {
"AuthorName": authorname,
"AuthorID": authorid,
"AuthorLink": "",
"BookName": bookname,
"BookSub": booksub,
"BookDesc": bookdesc,
"BookIsbn": bookisbn,
"BookPub": bookpub,
"BookGenre": bookgenre,
"BookImg": bookimg,
"BookLink": booklink,
"BookRate": bookrate,
"BookPages": bookpages,
"BookDate": bookdate,
"BookLang": booklang,
"Status": book_status,
"BookAdded": today(),
"Series": series,
"SeriesNum": seriesNum
}
resultcount = resultcount + 1
myDB.upsert("books", newValueDict, controlValueDict)
logger.debug(u"Book found: " + bookname + " " + bookdate)
if 'nocover' in bookimg or 'nophoto' in bookimg:
# try to get a cover from librarything
workcover = getBookCover(bookid)
if workcover:
logger.debug(u'Updated cover for %s to %s' % (bookname, workcover))
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": workcover}
myDB.upsert("books", newValueDict, controlValueDict)
elif bookimg and bookimg.startswith('http'):
link = cache_cover(bookid, bookimg)
if link is not None:
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": link}
myDB.upsert("books", newValueDict, controlValueDict)
if seriesNum is None:
# try to get series info from librarything
series, seriesNum = getWorkSeries(bookid)
if seriesNum:
logger.debug(u'Updated series: %s [%s]' % (series, seriesNum))
controlValueDict = {"BookID": bookid}
newValueDict = {
"Series": series,
"SeriesNum": seriesNum
}
myDB.upsert("books", newValueDict, controlValueDict)
worklink = getWorkPage(bookid)
if worklink:
controlValueDict = {"BookID": bookid}
newValueDict = {"WorkPage": worklink}
myDB.upsert("books", newValueDict, controlValueDict)
if not find_book_status:
logger.debug("[%s] Added book: %s [%s]" % (authorname, bookname, booklang))
added_count = added_count + 1
else:
updated_count = updated_count + 1
logger.debug("[%s] Updated book: %s" % (authorname, bookname))
else:
book_ignore_count = book_ignore_count + 1
except KeyError:
pass
logger.debug('[%s] The Google Books API was hit %s time%s to populate book list' %
(authorname, api_hits, plural(api_hits)))
lastbook = myDB.match('SELECT BookName, BookLink, BookDate from books WHERE AuthorID="%s" \
AND Status != "Ignored" order by BookDate DESC' % authorid)
if lastbook: # maybe there are no books [remaining] for this author
lastbookname = lastbook['BookName']
lastbooklink = lastbook['BookLink']
lastbookdate = lastbook['BookDate']
else:
lastbookname = None
lastbooklink = None
lastbookdate = None
controlValueDict = {"AuthorID": authorid}
newValueDict = {
"Status": "Active",
"LastBook": lastbookname,
"LastLink": lastbooklink,
"LastDate": lastbookdate
}
myDB.upsert("authors", newValueDict, controlValueDict)
logger.debug("Found %s total book%s for author" % (total_count, plural(total_count)))
logger.debug("Removed %s bad language result%s for author" % (ignored, plural(ignored)))
logger.debug(
"Removed %s bad character or no-name result%s for author" %
(removedResults, plural(removedResults)))
logger.debug("Removed %s duplicate result%s for author" % (duplicates, plural(duplicates)))
logger.debug("Ignored %s book%s by author marked as Ignored" % (book_ignore_count, plural(book_ignore_count)))
logger.debug("Imported/Updated %s book%s for author" % (resultcount, plural(resultcount)))
myDB.action('insert into stats values ("%s", %i, %i, %i, %i, %i, %i, %i, %i, %i)' %
(authorname.replace('"', '""'), api_hits, gr_lang_hits, lt_lang_hits, gb_lang_change, cache_hits,
ignored, removedResults, not_cached, duplicates))
if refresh:
logger.info("[%s] Book processing complete: Added %s book%s / Updated %s book%s" %
(authorname, added_count, plural(added_count), updated_count, plural(updated_count)))
else:
logger.info("[%s] Book processing complete: Added %s book%s to the database" %
(authorname, added_count, plural(added_count)))
return books_dict
def find_book(self, bookid=None, queue=None):
myDB = database.DBConnection()
if not lazylibrarian.GB_API:
logger.warn('No GoogleBooks API key, check config')
URL = 'https://www.googleapis.com/books/v1/volumes/' + \
str(bookid) + "?key=" + lazylibrarian.GB_API
jsonresults, in_cache = get_json_request(URL)
if jsonresults is None:
logger.debug('No results found for %s' % bookname)
return
bookname = jsonresults['volumeInfo']['title']
dic = {':': '', '"': '', '\'': ''}
bookname = replace_all(bookname, dic)
bookname = unaccented(bookname)
bookname = bookname.strip() # strip whitespace
try:
authorname = jsonresults['volumeInfo']['authors'][0]
except KeyError:
logger.debug(
'Book %s does not contain author field, skipping' %
bookname)
return
try:
# warn if language is in ignore list, but user said they wanted
# this book
booklang = jsonresults['volumeInfo']['language']
valid_langs = ([valid_lang.strip()
for valid_lang in lazylibrarian.IMP_PREFLANG.split(',')])
if booklang not in valid_langs:
logger.debug(
'Book %s language does not match preference' %
bookname)
except KeyError:
logger.debug('Book does not have language field')
booklang = "Unknown"
try:
bookpub = jsonresults['volumeInfo']['publisher']
except KeyError:
bookpub = None
series = None
seriesNum = None
try:
booksub = jsonresults['volumeInfo']['subtitle']
try:
series = booksub.split('(')[1].split(' Series ')[0]
except IndexError:
series = None
try:
seriesNum = booksub.split('(')[1].split(' Series ')[1].split(')')[0]
if seriesNum[0] == '#':
seriesNum = seriesNum[1:]
except IndexError:
seriesNum = None
except KeyError:
booksub = None
try:
bookdate = jsonresults['volumeInfo']['publishedDate']
except KeyError:
bookdate = '0000-00-00'
try:
bookimg = jsonresults['volumeInfo']['imageLinks']['thumbnail']
except KeyError:
bookimg = 'images/nocover.png'
try:
bookrate = jsonresults['volumeInfo']['averageRating']
except KeyError:
bookrate = 0
try:
bookpages = jsonresults['volumeInfo']['pageCount']
except KeyError:
bookpages = 0
try:
bookgenre = jsonresults['volumeInfo']['categories'][0]
except KeyError:
bookgenre = None
try:
bookdesc = jsonresults['volumeInfo']['description']
except KeyError:
bookdesc = None
try:
if jsonresults['volumeInfo']['industryIdentifiers'][0]['type'] == 'ISBN_10':
bookisbn = jsonresults['volumeInfo'][
'industryIdentifiers'][0]['identifier']
else:
bookisbn = None
except KeyError:
bookisbn = None
booklink = jsonresults['volumeInfo']['canonicalVolumeLink']
bookrate = float(bookrate)
name = jsonresults['volumeInfo']['authors'][0]
GR = GoodReads(name)
author = GR.find_author_id()
if author:
AuthorID = author['authorid']
controlValueDict = {"BookID": bookid}
newValueDict = {
"AuthorName": authorname,
"AuthorID": AuthorID,
"AuthorLink": "",
"BookName": bookname,
"BookSub": booksub,
"BookDesc": bookdesc,
"BookIsbn": bookisbn,
"BookPub": bookpub,
"BookGenre": bookgenre,
"BookImg": bookimg,
"BookLink": booklink,
"BookRate": bookrate,
"BookPages": bookpages,
"BookDate": bookdate,
"BookLang": booklang,
"Status": "Wanted",
"BookAdded": today(),
"Series": series,
"SeriesNum": seriesNum
}
myDB.upsert("books", newValueDict, controlValueDict)
logger.debug("%s added to the books database" % bookname)
if 'nocover' in bookimg or 'nophoto' in bookimg:
# try to get a cover from librarything
workcover = getBookCover(bookid)
if workcover:
logger.debug(u'Updated cover for %s to %s' % (bookname, workcover))
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": workcover}
myDB.upsert("books", newValueDict, controlValueDict)
elif bookimg and bookimg.startswith('http'):
link = cache_cover(bookid, bookimg)
if link:
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": link}
myDB.upsert("books", newValueDict, controlValueDict)
if seriesNum is None:
# try to get series info from librarything
series, seriesNum = getWorkSeries(bookid)
if seriesNum:
logger.debug(u'Updated series: %s [%s]' % (series, seriesNum))
controlValueDict = {"BookID": bookid}
newValueDict = {
"Series": series,
"SeriesNum": seriesNum
}
myDB.upsert("books", newValueDict, controlValueDict)
worklink = getWorkPage(bookid)
if worklink:
controlValueDict = {"BookID": bookid}
newValueDict = {"WorkPage": worklink}
myDB.upsert("books", newValueDict, controlValueDict)
def get_author_books(self, authorid=None, authorname=None, refresh=False):
try:
logger.debug('[%s] Now processing books with Google Books API' % authorname)
# google doesnt like accents in author names
set_url = self.url + urllib.quote('inauthor:"%s"' % unaccented_str(authorname))
URL = set_url + '&' + urllib.urlencode(self.params)
api_hits = 0
gr_lang_hits = 0
lt_lang_hits = 0
gb_lang_change = 0
cache_hits = 0
not_cached = 0
# Artist is loading
myDB = database.DBConnection()
controlValueDict = {"AuthorID": authorid}
newValueDict = {"Status": "Loading"}
myDB.upsert("authors", newValueDict, controlValueDict)
try:
startindex = 0
resultcount = 0
removedResults = 0
duplicates = 0
ignored = 0
added_count = 0
updated_count = 0
book_ignore_count = 0
total_count = 0
number_results = 1
valid_langs = ([valid_lang.strip()
for valid_lang in lazylibrarian.IMP_PREFLANG.split(',')])
while startindex < number_results:
self.params['startIndex'] = startindex
URL = set_url + '&' + urllib.urlencode(self.params)
try:
jsonresults, in_cache = get_json_request(URL, useCache=not refresh)
if jsonresults is None:
number_results = 0
else:
if not in_cache:
api_hits = api_hits + 1
number_results = jsonresults['totalItems']
except HTTPError as err:
logger.warn(
'Google Books API Error [%s]: Check your API key or wait a while' %
err.reason)
break
if number_results == 0:
logger.warn('Found no results for %s' % authorname)
break
else:
logger.debug('Found %s result%s for %s' % (number_results, plural(number_results), authorname))
startindex = startindex + 40
for item in jsonresults['items']:
total_count = total_count + 1
# skip if no author, no author is no book.
try:
Author = item['volumeInfo']['authors'][0]
except KeyError:
logger.debug('Skipped a result without authorfield.')
continue
try:
if item['volumeInfo']['industryIdentifiers'][0]['type'] == 'ISBN_10':
bookisbn = item['volumeInfo'][
'industryIdentifiers'][0]['identifier']
else:
bookisbn = ""
except KeyError:
bookisbn = ""
isbnhead = ""
if len(bookisbn) == 10:
isbnhead = bookisbn[0:3]
try:
booklang = item['volumeInfo']['language']
except KeyError:
booklang = "Unknown"
# do we care about language?
if "All" not in valid_langs:
if bookisbn != "":
# seems google lies to us, sometimes tells us books
# are in english when they are not
if booklang == "Unknown" or booklang == "en":
googlelang = booklang
match = myDB.match('SELECT lang FROM languages where isbn = "%s"' %
(isbnhead))
if match:
booklang = match['lang']
cache_hits = cache_hits + 1
logger.debug(
"Found cached language [%s] for [%s]" %
(booklang, isbnhead))
else:
# no match in cache, try searching librarything for a language code using the isbn
# if no language found, librarything return value is "invalid" or "unknown"
# librarything returns plain text, not xml
BOOK_URL = 'http://www.librarything.com/api/thingLang.php?isbn=' + \
bookisbn
try:
librarything_wait()
resp = urllib2.urlopen(BOOK_URL, timeout=30).read()
lt_lang_hits = lt_lang_hits + 1
logger.debug(
"LibraryThing reports language [%s] for %s" % (resp, isbnhead))
if (resp != 'invalid' and resp != 'unknown'):
booklang = resp # found a language code
myDB.action('insert into languages values ("%s", "%s")' %
(isbnhead, booklang))
logger.debug(u"LT language: " + booklang)
except Exception as e:
booklang = ""
logger.error("Error finding language: %s" % str(e))
if googlelang == "en" and booklang not in ["en-US", "en-GB", "eng"]:
# these are all english, may need to expand
# this list
booknamealt = item['volumeInfo']['title']
logger.debug("%s Google thinks [%s], we think [%s]" %
(booknamealt, googlelang, booklang))
gb_lang_change = gb_lang_change + 1
else:
match = myDB.match('SELECT lang FROM languages where isbn = "%s"' %
(isbnhead))
if not match:
myDB.action(
'insert into languages values ("%s", "%s")' %
(isbnhead, booklang))
logger.debug(u"GB language: " + booklang)
# skip if language is in ignore list
if booklang not in valid_langs:
booknamealt = item['volumeInfo']['title']
logger.debug(
'Skipped [%s] with language %s' %
(booknamealt, booklang))
ignored = ignored + 1
continue
try:
bookpub = item['volumeInfo']['publisher']
except KeyError:
bookpub = None
try:
booksub = item['volumeInfo']['subtitle']
except KeyError:
booksub = None
if booksub is None:
series = None
seriesNum = None
else:
try:
series = booksub.split('(')[1].split(' Series ')[0]
except IndexError:
series = None
try:
seriesNum = booksub.split('(')[1].split(' Series ')[1].split(')')[0]
if seriesNum[0] == '#':
seriesNum = seriesNum[1:]
except IndexError:
seriesNum = None
try:
bookdate = item['volumeInfo']['publishedDate']
except KeyError:
bookdate = '0000-00-00'
try:
bookimg = item['volumeInfo']['imageLinks']['thumbnail']
except KeyError:
bookimg = 'images/nocover.png'
try:
bookrate = item['volumeInfo']['averageRating']
except KeyError:
bookrate = 0
try:
bookpages = item['volumeInfo']['pageCount']
except KeyError:
bookpages = 0
try:
bookgenre = item['volumeInfo']['categories'][0]
except KeyError:
bookgenre = None
try:
bookdesc = item['volumeInfo']['description']
except KeyError:
bookdesc = None
bookname = item['volumeInfo']['title']
bookname = unaccented(bookname)
dic = {':': '', '"': '', '\'': ''}
bookname = replace_all(bookname, dic)
bookname = bookname.strip() # strip whitespace
booklink = item['volumeInfo']['canonicalVolumeLink']
bookrate = float(bookrate)
bookid = item['id']
# GoodReads sometimes has multiple bookids for the same book (same author/title, different editions)
# and sometimes uses the same bookid if the book is the same but the title is slightly different
#
# Not sure if googlebooks does too, but we only want one...
find_book_status = myDB.select('SELECT * FROM books WHERE BookID = "%s"' % bookid)
if find_book_status:
for resulted in find_book_status:
book_status = resulted['Status']
locked = resulted['Manual']
else:
book_status = lazylibrarian.NEWBOOK_STATUS
locked = False
rejected = False
if re.match('[^\w-]', bookname): # remove books with bad characters in title
logger.debug("[%s] removed book for bad characters" % bookname)
removedResults = removedResults + 1
rejected = True
if not rejected and not bookname:
logger.debug('Rejecting bookid %s for %s, no bookname' %
(bookid, authorname))
removedResults = removedResults + 1
rejected = True
if not rejected:
find_books = myDB.select('SELECT * FROM books WHERE BookName = "%s" and AuthorName = "%s"' %
(bookname.replace('"', '""'), authorname.replace('"', '""')))
if find_books:
for find_book in find_books:
if find_book['BookID'] != bookid:
# we have a book with this author/title already
logger.debug('Rejecting bookid %s for [%s][%s] already got %s' %
(find_book['BookID'], authorname, bookname, bookid))
rejected = True
duplicates = duplicates + 1
if not rejected:
find_books = myDB.match('SELECT AuthorName,BookName FROM books WHERE BookID = "%s"' % bookid)
if find_books:
# we have a book with this bookid already
if bookname != find_books['BookName'] or authorNameResult != find_books['AuthorName']:
logger.debug('Rejecting bookid %s for [%s][%s] already got bookid for [%s][%s]' %
(bookid, authorname, bookname,
find_books['AuthorName'], find_books['BookName']))
else:
logger.debug('Rejecting bookid %s for [%s][%s] already got this book in database' %
(bookid, authorname, bookname))
duplicates = duplicates + 1
rejected = True
if not rejected:
if book_status != "Ignored" and not locked:
controlValueDict = {"BookID": bookid}
newValueDict = {
"AuthorName": authorname,
"AuthorID": authorid,
"AuthorLink": "",
"BookName": bookname,
"BookSub": booksub,
"BookDesc": bookdesc,
"BookIsbn": bookisbn,
"BookPub": bookpub,
"BookGenre": bookgenre,
"BookImg": bookimg,
"BookLink": booklink,
"BookRate": bookrate,
"BookPages": bookpages,
"BookDate": bookdate,
"BookLang": booklang,
"Status": book_status,
"BookAdded": today(),
"Series": series,
"SeriesNum": seriesNum
}
resultcount = resultcount + 1
myDB.upsert("books", newValueDict, controlValueDict)
logger.debug(u"Book found: " + bookname + " " + bookdate)
if 'nocover' in bookimg or 'nophoto' in bookimg:
# try to get a cover from librarything
workcover = getBookCover(bookid)
if workcover:
logger.debug(u'Updated cover for %s to %s' % (bookname, workcover))
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": workcover}
myDB.upsert("books", newValueDict, controlValueDict)
elif bookimg and bookimg.startswith('http'):
link = cache_cover(bookid, bookimg)
if link:
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": link}
myDB.upsert("books", newValueDict, controlValueDict)
if seriesNum is None:
# try to get series info from librarything
series, seriesNum = getWorkSeries(bookid)
if seriesNum:
logger.debug(u'Updated series: %s [%s]' % (series, seriesNum))
controlValueDict = {"BookID": bookid}
newValueDict = {
"Series": series,
"SeriesNum": seriesNum
}
myDB.upsert("books", newValueDict, controlValueDict)
worklink = getWorkPage(bookid)
if worklink:
controlValueDict = {"BookID": bookid}
newValueDict = {"WorkPage": worklink}
myDB.upsert("books", newValueDict, controlValueDict)
if not find_book_status:
logger.debug("[%s] Added book: %s [%s]" % (authorname, bookname, booklang))
added_count = added_count + 1
else:
updated_count = updated_count + 1
logger.debug("[%s] Updated book: %s" % (authorname, bookname))
else:
book_ignore_count = book_ignore_count + 1
except KeyError:
pass
logger.debug('[%s] The Google Books API was hit %s time%s to populate book list' %
(authorname, api_hits, plural(api_hits)))
lastbook = myDB.match('SELECT BookName, BookLink, BookDate from books WHERE AuthorID="%s" \
AND Status != "Ignored" order by BookDate DESC' % authorid)
if lastbook: # maybe there are no books [remaining] for this author
lastbookname = lastbook['BookName']
lastbooklink = lastbook['BookLink']
lastbookdate = lastbook['BookDate']
else:
lastbookname = None
lastbooklink = None
lastbookdate = None
controlValueDict = {"AuthorID": authorid}
newValueDict = {
"Status": "Active",
"LastBook": lastbookname,
"LastLink": lastbooklink,
"LastDate": lastbookdate
}
myDB.upsert("authors", newValueDict, controlValueDict)
logger.debug("Found %s total book%s for author" % (total_count, plural(total_count)))
logger.debug("Removed %s unwanted language result%s for author" % (ignored, plural(ignored)))
logger.debug(
"Removed %s bad character or no-name result%s for author" %
(removedResults, plural(removedResults)))
logger.debug("Removed %s duplicate result%s for author" % (duplicates, plural(duplicates)))
logger.debug("Found %s book%s by author marked as Ignored" % (book_ignore_count, plural(book_ignore_count)))
logger.debug("Imported/Updated %s book%s for author" % (resultcount, plural(resultcount)))
myDB.action('insert into stats values ("%s", %i, %i, %i, %i, %i, %i, %i, %i, %i)' %
(authorname.replace('"', '""'), api_hits, gr_lang_hits, lt_lang_hits, gb_lang_change, cache_hits,
ignored, removedResults, not_cached, duplicates))
if refresh:
logger.info("[%s] Book processing complete: Added %s book%s / Updated %s book%s" %
(authorname, added_count, plural(added_count), updated_count, plural(updated_count)))
else:
logger.info("[%s] Book processing complete: Added %s book%s to the database" %
(authorname, added_count, plural(added_count)))
except Exception as e:
logger.error('Unhandled exception in GB.get_author_books: %s' % traceback.format_exc())
def LibraryScan(startdir=None):
""" Scan a directory tree adding new books into database
Return how many books you added """
if not startdir:
if not lazylibrarian.DESTINATION_DIR:
return 0
else:
startdir = lazylibrarian.DESTINATION_DIR
if not os.path.isdir(startdir):
logger.warn('Cannot find directory: %s. Not scanning' % startdir)
return 0
myDB = database.DBConnection()
# keep statistics of full library scans
if startdir == lazylibrarian.DESTINATION_DIR:
myDB.action('DELETE from stats')
logger.info('Scanning ebook directory: %s' % startdir)
new_book_count = 0
modified_count = 0
file_count = 0
author = ""
if lazylibrarian.FULL_SCAN and startdir == lazylibrarian.DESTINATION_DIR:
books = myDB.select(
'select AuthorName, BookName, BookFile, BookID from books where Status="Open"'
)
status = lazylibrarian.NOTFOUND_STATUS
logger.info('Missing books will be marked as %s' % status)
for book in books:
bookName = book['BookName']
bookAuthor = book['AuthorName']
bookID = book['BookID']
bookfile = book['BookFile']
if not (bookfile and os.path.isfile(bookfile)):
myDB.action('update books set Status="%s" where BookID="%s"' %
(status, bookID))
myDB.action('update books set BookFile="" where BookID="%s"' %
bookID)
logger.warn('Book %s - %s updated as not found on disk' %
(bookAuthor, bookName))
# to save repeat-scans of the same directory if it contains multiple formats of the same book,
# keep track of which directories we've already looked at
processed_subdirectories = []
matchString = ''
for char in lazylibrarian.EBOOK_DEST_FILE:
matchString = matchString + '\\' + char
# massage the EBOOK_DEST_FILE config parameter into something we can use
# with regular expression matching
booktypes = ''
count = -1
booktype_list = getList(lazylibrarian.EBOOK_TYPE)
for book_type in booktype_list:
count += 1
if count == 0:
booktypes = book_type
else:
booktypes = booktypes + '|' + book_type
matchString = matchString.replace(
"\\$\\A\\u\\t\\h\\o\\r", "(?P<author>.*?)").replace(
"\\$\\T\\i\\t\\l\\e", "(?P<book>.*?)") + '\.[' + booktypes + ']'
pattern = re.compile(matchString, re.VERBOSE)
for r, d, f in os.walk(startdir):
for directory in d[:]:
# prevent magazine being scanned
if directory.startswith("_") or directory.startswith("."):
d.remove(directory)
for files in f:
file_count += 1
if isinstance(r, str):
r = r.decode(lazylibrarian.SYS_ENCODING)
subdirectory = r.replace(startdir, '')
# Added new code to skip if we've done this directory before.
# Made this conditional with a switch in config.ini
# in case user keeps multiple different books in the same subdirectory
if (lazylibrarian.IMP_SINGLEBOOK) and (
subdirectory in processed_subdirectories):
logger.debug("[%s] already scanned" % subdirectory)
else:
# If this is a book, try to get author/title/isbn/language
# if epub or mobi, read metadata from the book
# If metadata.opf exists, use that allowing it to override
# embedded metadata. User may have edited metadata.opf
# to merge author aliases together
# If all else fails, try pattern match for author/title
# and look up isbn/lang from LT or GR later
match = 0
if is_valid_booktype(files):
logger.debug("[%s] Now scanning subdirectory %s" %
(startdir, subdirectory))
language = "Unknown"
isbn = ""
book = ""
author = ""
extn = os.path.splitext(files)[1]
# if it's an epub or a mobi we can try to read metadata from it
if (extn == ".epub") or (extn == ".mobi"):
book_filename = os.path.join(
r.encode(lazylibrarian.SYS_ENCODING),
files.encode(lazylibrarian.SYS_ENCODING))
try:
res = get_book_info(book_filename)
except Exception as e:
logger.debug('get_book_info failed for %s, %s' %
(book_filename, str(e)))
res = {}
if 'title' in res and 'creator' in res: # this is the minimum we need
match = 1
book = res['title']
author = res['creator']
if 'language' in res:
language = res['language']
if 'identifier' in res:
isbn = res['identifier']
if 'type' in res:
extn = res['type']
logger.debug("book meta [%s] [%s] [%s] [%s] [%s]" %
(isbn, language, author, book, extn))
else:
logger.debug("Book meta incomplete in %s" %
book_filename)
# calibre uses "metadata.opf", LL uses "bookname - authorname.opf"
# just look for any .opf file in the current directory since we don't know
# LL preferred authorname/bookname at this point.
# Allow metadata in file to override book contents as may be users pref
metafile = opf_file(r)
try:
res = get_book_info(metafile)
except Exception as e:
logger.debug('get_book_info failed for %s, %s' %
(metafile, str(e)))
res = {}
if 'title' in res and 'creator' in res: # this is the minimum we need
match = 1
book = res['title']
author = res['creator']
if 'language' in res:
language = res['language']
if 'identifier' in res:
isbn = res['identifier']
logger.debug("file meta [%s] [%s] [%s] [%s]" %
(isbn, language, author, book))
else:
logger.debug("File meta incomplete in %s" % metafile)
if not match: # no author/book from metadata file, and not embedded either
match = pattern.match(files)
if match:
author = match.group("author")
book = match.group("book")
else:
logger.debug("Pattern match failed [%s]" % files)
if match:
# flag that we found a book in this subdirectory
processed_subdirectories.append(subdirectory)
# If we have a valid looking isbn, and language != "Unknown", add it to cache
if language != "Unknown" and is_valid_isbn(isbn):
logger.debug("Found Language [%s] ISBN [%s]" %
(language, isbn))
# we need to add it to language cache if not already
# there, is_valid_isbn has checked length is 10 or 13
if len(isbn) == 10:
isbnhead = isbn[0:3]
else:
isbnhead = isbn[3:6]
match = myDB.match(
'SELECT lang FROM languages where isbn = "%s"'
% (isbnhead))
if not match:
myDB.action(
'insert into languages values ("%s", "%s")'
% (isbnhead, language))
logger.debug("Cached Lang [%s] ISBN [%s]" %
(language, isbnhead))
else:
logger.debug(
"Already cached Lang [%s] ISBN [%s]" %
(language, isbnhead))
# get authors name in a consistent format
if "," in author: # "surname, forename"
words = author.split(',')
author = words[1].strip() + ' ' + words[0].strip(
) # "forename surname"
if author[1] == ' ':
author = author.replace(' ', '.')
author = author.replace('..', '.')
# Check if the author exists, and import the author if not,
# before starting any complicated book-name matching to save repeating the search
#
check_exist_author = myDB.match(
'SELECT * FROM authors where AuthorName="%s"' %
author.replace('"', '""'))
if not check_exist_author and lazylibrarian.ADD_AUTHOR:
# no match for supplied author, but we're allowed to
# add new ones
GR = GoodReads(author)
try:
author_gr = GR.find_author_id()
except Exception as e:
logger.warn(
"Error finding author id for [%s] %s" %
(author, str(e)))
continue
# only try to add if GR data matches found author data
if author_gr:
authorname = author_gr['authorname']
# "J.R.R. Tolkien" is the same person as "J. R. R. Tolkien" and "J R R Tolkien"
match_auth = author.replace('.', '_')
match_auth = match_auth.replace(' ', '_')
match_auth = match_auth.replace('__', '_')
match_name = authorname.replace('.', '_')
match_name = match_name.replace(' ', '_')
match_name = match_name.replace('__', '_')
match_name = unaccented(match_name)
match_auth = unaccented(match_auth)
# allow a degree of fuzziness to cater for different accented character handling.
# some author names have accents,
# filename may have the accented or un-accented version of the character
# The currently non-configurable value of fuzziness might need to go in config
# We stored GoodReads unmodified author name in
# author_gr, so store in LL db under that
# fuzz.ratio doesn't lowercase for us
match_fuzz = fuzz.ratio(
match_auth.lower(), match_name.lower())
if match_fuzz < 90:
logger.debug(
"Failed to match author [%s] fuzz [%d]"
% (author, match_fuzz))
logger.debug(
"Failed to match author [%s] to authorname [%s]"
% (match_auth, match_name))
# To save loading hundreds of books by unknown
# authors at GR or GB, ignore if author "Unknown"
if (author != "Unknown") and (match_fuzz >=
90):
# use "intact" name for author that we stored in
# GR author_dict, not one of the various mangled versions
# otherwise the books appear to be by a different author!
author = author_gr['authorname']
# this new authorname may already be in the
# database, so check again
check_exist_author = myDB.match(
'SELECT * FROM authors where AuthorName="%s"'
% author.replace('"', '""'))
if not check_exist_author:
logger.info("Adding new author [%s]" %
author)
try:
addAuthorToDB(author)
check_exist_author = myDB.match(
'SELECT * FROM authors where AuthorName="%s"'
% author.replace('"', '""'))
except Exception:
continue
# check author exists in db, either newly loaded or already there
if not check_exist_author:
logger.debug(
"Failed to match author [%s] in database" %
author)
else:
# author exists, check if this book by this author is in our database
# metadata might have quotes in book name
book = book.replace("'", "")
bookid = find_book_in_db(myDB, author, book)
if bookid:
check_status = myDB.match(
'SELECT Status, BookFile from books where BookID="%s"'
% bookid)
if check_status['Status'] != 'Open':
# we found a new book
new_book_count += 1
myDB.action(
'UPDATE books set Status="Open" where BookID="%s"'
% bookid)
# update book location so we can check if it gets removed
# location may have changed since last scan
book_filename = os.path.join(r, files)
if book_filename != check_status['BookFile']:
modified_count += 1
logger.debug(
"Updating book location for %s %s" %
(author, book))
myDB.action(
'UPDATE books set BookFile="%s" where BookID="%s"'
% (book_filename, bookid))
# update cover file to cover.jpg in book folder (if exists)
bookdir = os.path.dirname(book_filename)
coverimg = os.path.join(bookdir, 'cover.jpg')
if os.path.isfile(coverimg):
cachedir = os.path.join(
str(lazylibrarian.PROG_DIR), 'data' +
os.sep + 'images' + os.sep + 'cache')
cacheimg = os.path.join(
cachedir, bookid + '.jpg')
copyfile(coverimg, cacheimg)
else:
logger.debug(
"Failed to match book [%s] by [%s] in database"
% (book, author))
logger.info("%s/%s new/modified book%s found and added to the database" %
(new_book_count, modified_count,
plural(new_book_count + modified_count)))
logger.info("%s file%s processed" % (file_count, plural(file_count)))
# show statistics of full library scans
if startdir == lazylibrarian.DESTINATION_DIR:
stats = myDB.match(
"SELECT sum(GR_book_hits), sum(GR_lang_hits), sum(LT_lang_hits), sum(GB_lang_change), \
sum(cache_hits), sum(bad_lang), sum(bad_char), sum(uncached), sum(duplicates) FROM stats"
)
if stats['sum(GR_book_hits)'] is not None:
# only show stats if new books added
if lazylibrarian.BOOK_API == "GoogleBooks":
logger.debug("GoogleBooks was hit %s time%s for books" %
(stats['sum(GR_book_hits)'],
plural(stats['sum(GR_book_hits)'])))
logger.debug("GoogleBooks language was changed %s time%s" %
(stats['sum(GB_lang_change)'],
plural(stats['sum(GB_lang_change)'])))
if lazylibrarian.BOOK_API == "GoodReads":
logger.debug("GoodReads was hit %s time%s for books" %
(stats['sum(GR_book_hits)'],
plural(stats['sum(GR_book_hits)'])))
logger.debug("GoodReads was hit %s time%s for languages" %
(stats['sum(GR_lang_hits)'],
plural(stats['sum(GR_lang_hits)'])))
logger.debug("LibraryThing was hit %s time%s for languages" %
(stats['sum(LT_lang_hits)'],
plural(stats['sum(LT_lang_hits)'])))
logger.debug(
"Language cache was hit %s time%s" %
(stats['sum(cache_hits)'], plural(stats['sum(cache_hits)'])))
logger.debug(
"Unwanted language removed %s book%s" %
(stats['sum(bad_lang)'], plural(stats['sum(bad_lang)'])))
logger.debug(
"Unwanted characters removed %s book%s" %
(stats['sum(bad_char)'], plural(stats['sum(bad_char)'])))
logger.debug(
"Unable to cache %s book%s with missing ISBN" %
(stats['sum(uncached)'], plural(stats['sum(uncached)'])))
logger.debug(
"Found %s duplicate book%s" %
(stats['sum(duplicates)'], plural(stats['sum(duplicates)'])))
logger.debug(
"Cache %s hit%s, %s miss" %
(lazylibrarian.CACHE_HIT, plural(
lazylibrarian.CACHE_HIT), lazylibrarian.CACHE_MISS))
cachesize = myDB.match(
"select count('ISBN') as counter from languages")
logger.debug("ISBN Language cache holds %s entries" %
cachesize['counter'])
nolang = len(
myDB.select(
'select BookID from Books where status="Open" and BookLang="Unknown"'
))
if nolang:
logger.warn(
"Found %s book%s in your library with unknown language" %
(nolang, plural(nolang)))
authors = myDB.select('select AuthorID from authors')
# Update bookcounts for all authors, not just new ones - refresh may have located
# new books for existing authors especially if switched provider gb/gr
else:
# single author/book import
authors = myDB.select(
'select AuthorID from authors where AuthorName = "%s"' %
author.replace('"', '""'))
logger.debug('Updating bookcounts for %i author%s' %
(len(authors), plural(len(authors))))
for author in authors:
update_totals(author['AuthorID'])
images = myDB.select(
'select bookid, bookimg, bookname from books where bookimg like "http%"'
)
if len(images):
logger.info("Caching cover%s for %i book%s" %
(plural(len(images)), len(images), plural(len(images))))
for item in images:
bookid = item['bookid']
bookimg = item['bookimg']
bookname = item['bookname']
newimg = cache_cover(bookid, bookimg)
if newimg is not None:
myDB.action('update books set BookImg="%s" where BookID="%s"' %
(newimg, bookid))
images = myDB.select(
'select AuthorID, AuthorImg, AuthorName from authors where AuthorImg like "http%"'
)
if len(images):
logger.info("Caching image%s for %i author%s" %
(plural(len(images)), len(images), plural(len(images))))
for item in images:
authorid = item['authorid']
authorimg = item['authorimg']
authorname = item['authorname']
newimg = cache_cover(authorid, authorimg)
if newimg is not None:
myDB.action(
'update authors set AuthorImg="%s" where AuthorID="%s"' %
(newimg, authorid))
setWorkPages()
logger.info('Library scan complete')
return new_book_count
def get_author_books(self, authorid=None, authorname=None, refresh=False):
api_hits = 0
gr_lang_hits = 0
lt_lang_hits = 0
gb_lang_change = 0
cache_hits = 0
not_cached = 0
URL = 'http://www.goodreads.com/author/list/' + authorid + '.xml?' + urllib.urlencode(self.params)
# Artist is loading
myDB = database.DBConnection()
controlValueDict = {"AuthorID": authorid}
newValueDict = {"Status": "Loading"}
myDB.upsert("authors", newValueDict, controlValueDict)
books_dict = []
try:
rootxml, in_cache = get_xml_request(URL, useCache=not refresh)
except Exception as e:
logger.error("Error fetching author books: %s" % e)
return books_dict
if rootxml is None:
logger.debug("Error requesting author books")
return books_dict
if not in_cache:
api_hits = api_hits + 1
resultxml = rootxml.getiterator('book')
valid_langs = ([valid_lang.strip() for valid_lang in lazylibrarian.IMP_PREFLANG.split(',')])
if not len(resultxml):
logger.warn('[%s] No books found for author with ID: %s' % (authorname, authorid))
else:
logger.debug("[%s] Now processing books with GoodReads API" % authorname)
resultsCount = 0
removedResults = 0
duplicates = 0
ignored = 0
added_count = 0
updated_count = 0
book_ignore_count = 0
total_count = 0
logger.debug(u"url " + URL)
authorNameResult = rootxml.find('./author/name').text
logger.debug(u"author name " + authorNameResult)
loopCount = 1
while resultxml is not None:
for book in resultxml:
total_count = total_count + 1
if (book.find('publication_year').text is None):
pubyear = "0000"
else:
pubyear = book.find('publication_year').text
try:
bookimg = book.find('image_url').text
if ('nocover' in bookimg):
bookimg = 'images/nocover.png'
except (KeyError,AttributeError):
bookimg = 'images/nocover.png'
# PAB this next section tries to get the book language using the isbn13 to look it up. If no isbn13 we skip the
# book entirely, rather than including it with an "Unknown" language. Changed this so we can still include the book
# with language set to "Unknown". There is a setting in config.ini to allow or skip books with "Unknown" language
# if you really don't want to include them.
# Not all GR books have isbn13 filled in, but all have a GR bookid, which we've already got, so use that.
# Also, with GR API rules we can only call the API once per second, which slows us down a lot when all we want
# is to get the language. We sleep for one second per book that GR knows about for each author you have in your
# library. The libraryThing API has the same 1 second restriction, and is limited to 1000 hits per day, but has
# fewer books with unknown language. To get around this and speed up the process, see if we already have a book
# in the database with a similar start to the ISBN. The way ISBNs work, digits 3-5 of a 13 char ISBN or digits 0-2
# of a 10 digit ISBN indicate the region/language so if two books have the same 3 digit isbn code, they _should_
# be the same language.
# I ran a simple python script on my library of 1500 books, and these codes were 100% correct on matching book
# languages, no mis-matches. It did result in a small number of books with "unknown" language being wrongly matched
# but most "unknown" were matched to the correct language.
# We could look up ISBNs we already know about in the database, but this only holds books in the languages we want
# to keep, which reduces the number of cache hits, so we create a new database table, holding ALL results including
# the ISBNs for languages we don't want and books we reject.
# The new table is created (if not exists) in init.py so by the time we get here there is an existing table.
# If we haven't an already matching partial ISBN, look up language code from libraryThing
# "http://www.librarything.com/api/thingLang.php?isbn=1234567890"
# If you find a matching language, add it to the database. If "unknown" or "invalid", try GR as maybe GR can
# provide a match.
# If both LT and GR return unknown, add isbn to db as "unknown". No point in repeatedly asking LT for a code
# it's told you it doesn't know.
# As an extra option, if language includes "All" in config.ini, we can skip this whole section and process
# everything much faster by not querying for language at all.
# It does mean we include a lot of unwanted foreign translations in the database, but it's _much_ faster.
bookLanguage = "Unknown"
find_field = "id"
isbn = ""
isbnhead = ""
if "All" not in valid_langs: # do we care about language
if (book.find('isbn').text is not None):
find_field = "isbn"
isbn = book.find('isbn').text
isbnhead = isbn[0:3]
else:
if (book.find('isbn13').text is not None):
find_field = "isbn13"
isbn = book.find('isbn13').text
isbnhead = isbn[3:6]
if (find_field != 'id'): # isbn or isbn13 found
match = myDB.action('SELECT lang FROM languages where isbn = "%s"' %
(isbnhead)).fetchone()
if (match):
bookLanguage = match['lang']
cache_hits = cache_hits + 1
logger.debug("Found cached language [%s] for %s [%s]" %
(bookLanguage, find_field, isbnhead))
else:
# no match in cache, try searching librarything for a language code using the isbn
# if no language found, librarything return value is "invalid" or "unknown"
# returns plain text, not xml
BOOK_URL = 'http://www.librarything.com/api/thingLang.php?isbn=' + isbn
try:
librarything_wait()
resp = urllib2.urlopen(BOOK_URL, timeout=30).read()
lt_lang_hits = lt_lang_hits + 1
logger.debug("LibraryThing reports language [%s] for %s" % (resp, isbnhead))
if ('invalid' in resp or 'Unknown' in resp):
find_field = "id" # reset the field to force search on goodreads
else:
bookLanguage = resp # found a language code
myDB.action('insert into languages values ("%s", "%s")' %
(isbnhead, bookLanguage))
logger.debug(u"LT language %s: %s" % (isbnhead, bookLanguage))
except Exception as e:
logger.error("Error finding LT language result for [%s], %s" % (isbn, e))
find_field = "id" # reset the field to search on goodreads
if (find_field == 'id'):
# [or bookLanguage == "Unknown"] no earlier match, we'll have to search the goodreads api
try:
if (book.find(find_field).text is not None):
BOOK_URL = 'http://www.goodreads.com/book/show?id=' + \
book.find(find_field).text + '&' + urllib.urlencode(self.params)
logger.debug(u"Book URL: " + BOOK_URL)
try:
time_now = int(time.time())
if time_now <= lazylibrarian.LAST_GOODREADS:
time.sleep(1)
BOOK_rootxml, in_cache = get_xml_request(BOOK_URL)
if BOOK_rootxml is None:
logger.debug('Error requesting book language code')
bookLanguage = ""
else:
if not in_cache:
# only update last_goodreads if the result wasn't found in the cache
lazylibrarian.LAST_GOODREADS = time_now
bookLanguage = BOOK_rootxml.find('./book/language_code').text
except Exception as e:
logger.error("Error finding book results: %s" % e)
if not in_cache:
gr_lang_hits = gr_lang_hits + 1
if not bookLanguage:
bookLanguage = "Unknown"
if (isbnhead != ""):
# GR didn't give an isbn so we can't cache it, just use language for this book
myDB.action('insert into languages values ("%s", "%s")' %
(isbnhead, bookLanguage))
logger.debug("GoodReads reports language [%s] for %s" %
(bookLanguage, isbnhead))
else:
not_cached = not_cached + 1
logger.debug(u"GR language: " + bookLanguage)
else:
logger.debug("No %s provided for [%s]" % (find_field, book.find('title').text))
# continue
except Exception as e:
logger.debug(u"An error has occured: %s" % e)
if bookLanguage not in valid_langs:
logger.debug('Skipped a book with language %s' % bookLanguage)
ignored = ignored + 1
continue
bookname = book.find('title').text
bookid = book.find('id').text
bookdesc = book.find('description').text
bookisbn = book.find('isbn').text
bookpub = book.find('publisher').text
booklink = book.find('link').text
bookrate = float(book.find('average_rating').text)
bookpages = book.find('num_pages').text
bookname = unaccented(bookname)
if ': ' in bookname:
parts = bookname.split(': ', 1)
bookname = parts[0]
booksub = parts[1]
else:
booksub = ''
dic = {':': '', '"': '', '\'': ''}
bookname = replace_all(bookname, dic)
bookname = bookname.strip() # strip whitespace
booksub = replace_all(booksub, dic)
booksub = booksub.strip() # strip whitespace
if booksub:
series,seriesNum = bookSeries(booksub)
else:
series,seriesNum = bookSeries(bookname)
# GoodReads sometimes has multiple bookids for the same book (same author/title, different editions)
# and sometimes uses the same bookid if the book is the same but the title is slightly different
# We use bookid, then reject if another author/title has a different bookid so we just keep one...
find_book_status = myDB.select('SELECT * FROM books WHERE BookID = "%s"' % bookid)
if find_book_status:
for resulted in find_book_status:
book_status = resulted['Status']
locked = resulted ['Manual']
else:
book_status = lazylibrarian.NEWBOOK_STATUS
locked = False
rejected = False
if re.match('[^\w-]', bookname): # reject books with bad characters in title
logger.debug(u"removed result [" + bookname + "] for bad characters")
removedResults = removedResults + 1
rejected = True
if not rejected and not bookname:
logger.debug('Rejecting bookid %s for %s, no bookname' %
(bookid, authorNameResult))
removedResults = removedResults + 1
rejected = True
if not rejected:
find_books = myDB.select('SELECT * FROM books WHERE BookName = "%s" and AuthorName = "%s"' %
(bookname, authorNameResult))
if find_books:
for find_book in find_books:
if find_book['BookID'] != bookid:
# we have a book with this author/title already
logger.debug('Rejecting bookid %s for [%s][%s] already got %s' %
(find_book['BookID'], authorNameResult, bookname, bookid))
duplicates = duplicates + 1
rejected = True
break
if not rejected:
find_books = myDB.select('SELECT * FROM books WHERE BookID = "%s"' % bookid)
if find_books:
# we have a book with this bookid already
logger.debug('Rejecting bookid %s for [%s][%s] already got this bookid in database' %
(bookid, authorNameResult, bookname))
duplicates = duplicates + 1
rejected = True
break
if not rejected:
if book_status != "Ignored":
if not locked:
controlValueDict = {"BookID": bookid}
newValueDict = {
"AuthorName": authorNameResult,
"AuthorID": authorid,
"AuthorLink": None,
"BookName": bookname,
"BookSub": booksub,
"BookDesc": bookdesc,
"BookIsbn": bookisbn,
"BookPub": bookpub,
"BookGenre": None,
"BookImg": bookimg,
"BookLink": booklink,
"BookRate": bookrate,
"BookPages": bookpages,
"BookDate": pubyear,
"BookLang": bookLanguage,
"Status": book_status,
"BookAdded": today(),
"Series": series,
"SeriesNum": seriesNum
}
resultsCount = resultsCount + 1
myDB.upsert("books", newValueDict, controlValueDict)
logger.debug(u"Book found: " + book.find('title').text + " " + pubyear)
if 'nocover' in bookimg or 'nophoto' in bookimg:
# try to get a cover from librarything
workcover = getBookCover(bookid)
if workcover:
logger.debug(u'Updated cover for %s to %s' % (bookname, workcover))
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": workcover}
myDB.upsert("books", newValueDict, controlValueDict)
elif bookimg and bookimg.startswith('http'):
link = cache_cover(bookid, bookimg)
if link is not None:
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": link}
myDB.upsert("books", newValueDict, controlValueDict)
if seriesNum == None:
# try to get series info from librarything
series, seriesNum = getWorkSeries(bookid)
if seriesNum:
logger.debug(u'Updated series: %s [%s]' % (series, seriesNum))
controlValueDict = {"BookID": bookid}
newValueDict = {
"Series": series,
"SeriesNum": seriesNum
}
myDB.upsert("books", newValueDict, controlValueDict)
worklink = getWorkPage(bookid)
if worklink:
controlValueDict = {"BookID": bookid}
newValueDict = {"WorkPage": worklink}
myDB.upsert("books", newValueDict, controlValueDict)
if not find_book_status:
logger.debug(u"[%s] Added book: %s" % (authorname, bookname))
added_count = added_count + 1
else:
logger.debug(u"[%s] Updated book: %s" % (authorname, bookname))
updated_count = updated_count + 1
else:
book_ignore_count = book_ignore_count + 1
loopCount = loopCount + 1
URL = 'http://www.goodreads.com/author/list/' + authorid + '.xml?' + \
urllib.urlencode(self.params) + '&page=' + str(loopCount)
resultxml = None
try:
rootxml, in_cache = get_xml_request(URL, useCache=not refresh)
if rootxml is None:
logger.debug('Error requesting next page of results')
else:
resultxml = rootxml.getiterator('book')
if not in_cache:
api_hits = api_hits + 1
except Exception as e:
resultxml = None
logger.error("Error finding next page of results: %s" % e)
if resultxml is not None:
if all(False for book in resultxml): # returns True if iterator is empty
resultxml = None
lastbook = myDB.action('SELECT BookName, BookLink, BookDate from books WHERE AuthorID="%s" \
AND Status != "Ignored" order by BookDate DESC' % authorid).fetchone()
if lastbook:
lastbookname = lastbook['BookName']
lastbooklink = lastbook['BookLink']
lastbookdate = lastbook['BookDate']
else:
lastbookname = None
lastbooklink = None
lastbookdate = None
controlValueDict = {"AuthorID": authorid}
newValueDict = {
"Status": "Active",
"LastBook": lastbookname,
"LastLink": lastbooklink,
"LastDate": lastbookdate
}
myDB.upsert("authors", newValueDict, controlValueDict)
# This is here because GoodReads sometimes has several entries with the same BookID!
modified_count = added_count + updated_count
logger.debug("Found %s total book%s for author" % (total_count, plural(total_count)))
logger.debug("Removed %s bad language result%s for author" % (ignored, plural(ignored)))
logger.debug("Removed %s bad character or no-name result%s for author" % (removedResults, plural(removedResults)))
logger.debug("Removed %s duplicate result%s for author" % (duplicates, plural(duplicates)))
logger.debug("Ignored %s book%s by author marked as Ignored" % (book_ignore_count, plural(book_ignore_count)))
logger.debug("Imported/Updated %s book%s for author" % (modified_count, plural(modified_count)))
myDB.action('insert into stats values ("%s", %i, %i, %i, %i, %i, %i, %i, %i, %i)' %
(authorname, api_hits, gr_lang_hits, lt_lang_hits, gb_lang_change,
cache_hits, ignored, removedResults, not_cached, duplicates))
if refresh:
logger.info("[%s] Book processing complete: Added %s book%s / Updated %s book%s" %
(authorname, added_count, plural(added_count), updated_count, plural(updated_count)))
else:
logger.info("[%s] Book processing complete: Added %s book%s to the database" %
(authorname, added_count, plural(added_count)))
return books_dict
def addAuthorToDB(authorname=None, refresh=False):
"""
Add an author to the database, and get list of all their books
If author already exists in database, refresh their details and booklist
"""
try:
myDB = database.DBConnection()
GR = GoodReads(authorname)
query = "SELECT * from authors WHERE AuthorName='%s'" % authorname.replace("'", "''")
dbauthor = myDB.match(query)
controlValueDict = {"AuthorName": authorname}
if not dbauthor:
newValueDict = {
"AuthorID": "0: %s" % (authorname),
"Status": "Loading"
}
logger.debug("Now adding new author: %s to database" % authorname)
else:
newValueDict = {"Status": "Loading"}
logger.debug("Now updating author: %s" % authorname)
myDB.upsert("authors", newValueDict, controlValueDict)
author = GR.find_author_id(refresh=refresh)
if author:
authorid = author['authorid']
authorlink = author['authorlink']
authorimg = author['authorimg']
controlValueDict = {"AuthorName": authorname}
newValueDict = {
"AuthorID": authorid,
"AuthorLink": authorlink,
"AuthorImg": authorimg,
"AuthorBorn": author['authorborn'],
"AuthorDeath": author['authordeath'],
"DateAdded": today(),
"Status": "Loading"
}
myDB.upsert("authors", newValueDict, controlValueDict)
else:
logger.warn(u"Nothing found for %s" % authorname)
myDB.action('DELETE from authors WHERE AuthorName="%s"' % authorname)
return
new_img = False
if authorimg and 'nophoto' in authorimg:
authorimg = getAuthorImage(authorid)
new_img = True
if authorimg and authorimg.startswith('http'):
newimg = cache_cover(authorid, authorimg)
if newimg:
authorimg = newimg
new_img = True
if new_img:
controlValueDict = {"AuthorID": authorid}
newValueDict = {"AuthorImg": authorimg}
myDB.upsert("authors", newValueDict, controlValueDict)
# process books
if lazylibrarian.BOOK_API == "GoogleBooks":
book_api = GoogleBooks()
book_api.get_author_books(authorid, authorname, refresh=refresh)
elif lazylibrarian.BOOK_API == "GoodReads":
GR.get_author_books(authorid, authorname, refresh=refresh)
# update totals works for existing authors only.
# New authors need their totals updating after libraryscan or import of books.
if dbauthor:
update_totals(authorid)
logger.debug("[%s] Author update complete" % authorname)
except Exception as e:
logger.error('Unhandled exception in addAuthorToDB: %s' % traceback.format_exc())
def getBookCover(bookID=None):
""" Return link to a local file containing a book cover image for a bookid.
Try 1. Local file cached from goodreads/googlebooks when book was imported
2. LibraryThing whatwork
3. Goodreads search if book was imported from goodreads
4. Google images search
Return None if no cover available. """
if not bookID:
logger.error("getBookCover- No bookID")
return None
cachedir = os.path.join(str(lazylibrarian.PROG_DIR),
'data' + os.sep + 'images' + os.sep + 'cache')
coverfile = os.path.join(cachedir, bookID + '.jpg')
if os.path.isfile(coverfile): # use cached image if there is one
lazylibrarian.CACHE_HIT = int(lazylibrarian.CACHE_HIT) + 1
logger.debug(u"getBookCover: Returning Cached response for %s" %
coverfile)
coverlink = 'images/cache/' + bookID + '.jpg'
return coverlink
lazylibrarian.CACHE_MISS = int(lazylibrarian.CACHE_MISS) + 1
work = getBookWork(bookID, "Cover")
if work:
try:
img = work.split('og:image')[1].split('="')[1].split('"')[0]
if img and img.startswith('http'):
coverlink = cache_cover(bookID, img)
if coverlink is not None:
logger.debug(
u"getBookCover: Caching librarything cover for %s" %
bookID)
return coverlink
else:
logger.debug(
"getBookCover: No image found in work page for %s" %
bookID)
except IndexError:
logger.debug('getBookCover: Image not found in work page for %s' %
bookID)
# not found in librarything work page, try to get a cover from goodreads or google instead
myDB = database.DBConnection()
item = myDB.match(
'select BookName,AuthorName,BookLink from books where bookID="%s"' %
bookID)
if item:
title = safe_unicode(item['BookName']).encode(
lazylibrarian.SYS_ENCODING)
author = safe_unicode(item['AuthorName']).encode(
lazylibrarian.SYS_ENCODING)
booklink = item['BookLink']
safeparams = urllib.quote_plus("%s %s" % (author, title))
if 'goodreads' in booklink:
# if the bookID is a goodreads one, we can call https://www.goodreads.com/book/show/{bookID}
# and scrape the page for og:image
# <meta property="og:image" content="https://i.gr-assets.com/images/S/photo.goodreads.com/books/1388267702i/16304._UY475_SS475_.jpg"/>
# to get the cover
time_now = int(time.time())
if time_now <= lazylibrarian.LAST_GOODREADS:
time.sleep(1)
lazylibrarian.LAST_GOODREADS = time_now
result, success = fetchURL(booklink)
if success:
try:
img = result.split('og:image')[1].split('="')[1].split(
'"')[0]
except IndexError:
img = None
if img and img.startswith(
'http'
) and 'nocover' not in img and 'nophoto' not in img:
time_now = int(time.time())
if time_now <= lazylibrarian.LAST_GOODREADS:
time.sleep(1)
lazylibrarian.LAST_GOODREADS = time_now
coverlink = cache_cover(bookID, img)
if coverlink is not None:
logger.debug(
"getBookCover: Caching goodreads cover for %s %s" %
(author, title))
return coverlink
else:
logger.debug(
"getBookCover: Error getting goodreads image for %s, [%s]"
% (img, result))
else:
logger.debug(
"getBookCover: No image found in goodreads page for %s"
% bookID)
else:
logger.debug("getBookCover: Error getting page %s, [%s]" %
(booklink, result))
# if this failed, try a google image search...
# tbm=isch search images
# tbs=isz:l large images
# ift:jpg jpeg file type
URL = "https://www.google.com/search?tbm=isch&tbs=isz:l,ift:jpg&as_q=" + safeparams + "+ebook"
result, success = fetchURL(URL)
if success:
try:
img = result.split('url?q=')[1].split('">')[1].split(
'src="')[1].split('"')[0]
except IndexError:
img = None
if img and img.startswith('http'):
coverlink = cache_cover(bookID, img)
if coverlink is not None:
logger.debug(
"getBookCover: Caching google cover for %s %s" %
(author, title))
return coverlink
else:
logger.debug(
"getBookCover: Error getting google image %s, [%s]" %
(img, result))
else:
logger.debug(
"getBookCover: No image found in google page for %s" %
bookID)
else:
logger.debug(
"getBookCover: Error getting google page for %s, [%s]" %
(safeparams, result))
return None
def LibraryScan(startdir=None):
""" Scan a directory tree adding new books into database
Return how many books you added """
if not startdir:
if not lazylibrarian.DESTINATION_DIR:
return 0
else:
startdir = lazylibrarian.DESTINATION_DIR
if not os.path.isdir(startdir):
logger.warn(
'Cannot find directory: %s. Not scanning' % startdir)
return 0
myDB = database.DBConnection()
# keep statistics of full library scans
if startdir == lazylibrarian.DESTINATION_DIR:
myDB.action('DELETE from stats')
logger.info('Scanning ebook directory: %s' % startdir)
new_book_count = 0
file_count = 0
author = ""
if lazylibrarian.FULL_SCAN and startdir == lazylibrarian.DESTINATION_DIR:
books = myDB.select(
'select AuthorName, BookName, BookFile, BookID from books where Status="Open"')
status = lazylibrarian.NOTFOUND_STATUS
logger.info('Missing books will be marked as %s' % status)
for book in books:
bookName = book['BookName']
bookAuthor = book['AuthorName']
bookID = book['BookID']
bookfile = book['BookFile']
if not(bookfile and os.path.isfile(bookfile)):
myDB.action('update books set Status="%s" where BookID="%s"' % (status, bookID))
myDB.action('update books set BookFile="" where BookID="%s"' % bookID)
logger.warn('Book %s - %s updated as not found on disk' % (bookAuthor, bookName))
# to save repeat-scans of the same directory if it contains multiple formats of the same book,
# keep track of which directories we've already looked at
processed_subdirectories = []
matchString = ''
for char in lazylibrarian.EBOOK_DEST_FILE:
matchString = matchString + '\\' + char
# massage the EBOOK_DEST_FILE config parameter into something we can use
# with regular expression matching
booktypes = ''
count = -1
booktype_list = getList(lazylibrarian.EBOOK_TYPE)
for book_type in booktype_list:
count += 1
if count == 0:
booktypes = book_type
else:
booktypes = booktypes + '|' + book_type
matchString = matchString.replace("\\$\\A\\u\\t\\h\\o\\r", "(?P<author>.*?)").replace(
"\\$\\T\\i\\t\\l\\e", "(?P<book>.*?)") + '\.[' + booktypes + ']'
pattern = re.compile(matchString, re.VERBOSE)
for r, d, f in os.walk(startdir):
for directory in d[:]:
# prevent magazine being scanned
if directory.startswith("_") or directory.startswith("."):
d.remove(directory)
for files in f:
file_count += 1
if isinstance(r, str):
r = r.decode(lazylibrarian.SYS_ENCODING)
subdirectory = r.replace(startdir, '')
# Added new code to skip if we've done this directory before.
# Made this conditional with a switch in config.ini
# in case user keeps multiple different books in the same subdirectory
if (lazylibrarian.IMP_SINGLEBOOK) and (subdirectory in processed_subdirectories):
logger.debug("[%s] already scanned" % subdirectory)
else:
# If this is a book, try to get author/title/isbn/language
# if epub or mobi, read metadata from the book
# If metadata.opf exists, use that allowing it to override
# embedded metadata. User may have edited metadata.opf
# to merge author aliases together
# If all else fails, try pattern match for author/title
# and look up isbn/lang from LT or GR later
match = 0
if is_valid_booktype(files):
logger.debug("[%s] Now scanning subdirectory %s" %
(startdir, subdirectory))
language = "Unknown"
isbn = ""
book = ""
author = ""
extn = os.path.splitext(files)[1]
# if it's an epub or a mobi we can try to read metadata from it
if (extn == ".epub") or (extn == ".mobi"):
book_filename = os.path.join(
r.encode(lazylibrarian.SYS_ENCODING), files.encode(lazylibrarian.SYS_ENCODING))
try:
res = get_book_info(book_filename)
except:
res = {}
if 'title' in res and 'creator' in res: # this is the minimum we need
match = 1
book = res['title']
author = res['creator']
if 'language' in res:
language = res['language']
if 'identifier' in res:
isbn = res['identifier']
if 'type' in res:
extn = res['type']
logger.debug("book meta [%s] [%s] [%s] [%s] [%s]" %
(isbn, language, author, book, extn))
else:
logger.debug("Book meta incomplete in %s" % book_filename)
# calibre uses "metadata.opf", LL uses "bookname - authorname.opf"
# just look for any .opf file in the current directory since we don't know
# LL preferred authorname/bookname at this point.
# Allow metadata in file to override book contents as may be users pref
metafile = opf_file(r)
try:
res = get_book_info(metafile)
except:
res = {}
if 'title' in res and 'creator' in res: # this is the minimum we need
match = 1
book = res['title']
author = res['creator']
if 'language' in res:
language = res['language']
if 'identifier' in res:
isbn = res['identifier']
logger.debug(
"file meta [%s] [%s] [%s] [%s]" %
(isbn, language, author, book))
else:
logger.debug("File meta incomplete in %s" % metafile)
if not match: # no author/book from metadata file, and not embedded either
match = pattern.match(files)
if match:
author = match.group("author")
book = match.group("book")
else:
logger.debug("Pattern match failed [%s]" % files)
if match:
# flag that we found a book in this subdirectory
processed_subdirectories.append(subdirectory)
# If we have a valid looking isbn, and language != "Unknown", add it to cache
if language != "Unknown" and is_valid_isbn(isbn):
logger.debug(
"Found Language [%s] ISBN [%s]" %
(language, isbn))
# we need to add it to language cache if not already
# there, is_valid_isbn has checked length is 10 or 13
if len(isbn) == 10:
isbnhead = isbn[0:3]
else:
isbnhead = isbn[3:6]
match = myDB.action(
'SELECT lang FROM languages where isbn = "%s"' %
(isbnhead)).fetchone()
if not match:
myDB.action(
'insert into languages values ("%s", "%s")' %
(isbnhead, language))
logger.debug(
"Cached Lang [%s] ISBN [%s]" %
(language, isbnhead))
else:
logger.debug(
"Already cached Lang [%s] ISBN [%s]" %
(language, isbnhead))
# get authors name in a consistent format
if "," in author: # "surname, forename"
words = author.split(',')
author = words[1].strip() + ' ' + words[0].strip() # "forename surname"
if author[1] == ' ':
author = author.replace(' ', '.')
author = author.replace('..', '.')
# Check if the author exists, and import the author if not,
# before starting any complicated book-name matching to save repeating the search
#
check_exist_author = myDB.action(
'SELECT * FROM authors where AuthorName="%s"' %
author).fetchone()
if not check_exist_author and lazylibrarian.ADD_AUTHOR:
# no match for supplied author, but we're allowed to
# add new ones
GR = GoodReads(author)
try:
author_gr = GR.find_author_id()
except:
logger.warn(
"Error finding author id for [%s]" %
author)
continue
# only try to add if GR data matches found author data
if author_gr:
authorname = author_gr['authorname']
# "J.R.R. Tolkien" is the same person as "J. R. R. Tolkien" and "J R R Tolkien"
match_auth = author.replace('.', '_')
match_auth = match_auth.replace(' ', '_')
match_auth = match_auth.replace('__', '_')
match_name = authorname.replace('.', '_')
match_name = match_name.replace(' ', '_')
match_name = match_name.replace('__', '_')
match_name = unaccented(match_name)
match_auth = unaccented(match_auth)
# allow a degree of fuzziness to cater for different accented character handling.
# some author names have accents,
# filename may have the accented or un-accented version of the character
# The currently non-configurable value of fuzziness might need to go in config
# We stored GoodReads unmodified author name in
# author_gr, so store in LL db under that
# fuzz.ratio doesn't lowercase for us
match_fuzz = fuzz.ratio(match_auth.lower(), match_name.lower())
if match_fuzz < 90:
logger.debug(
"Failed to match author [%s] fuzz [%d]" %
(author, match_fuzz))
logger.debug(
"Failed to match author [%s] to authorname [%s]" %
(match_auth, match_name))
# To save loading hundreds of books by unknown
# authors at GR or GB, ignore if author "Unknown"
if (author != "Unknown") and (match_fuzz >= 90):
# use "intact" name for author that we stored in
# GR author_dict, not one of the various mangled versions
# otherwise the books appear to be by a different author!
author = author_gr['authorname']
# this new authorname may already be in the
# database, so check again
check_exist_author = myDB.action(
'SELECT * FROM authors where AuthorName="%s"' %
author).fetchone()
if not check_exist_author:
logger.info(
"Adding new author [%s]" %
author)
try:
addAuthorToDB(author)
check_exist_author = myDB.action(
'SELECT * FROM authors where AuthorName="%s"' %
author).fetchone()
except:
continue
# check author exists in db, either newly loaded or already there
if not check_exist_author:
logger.debug(
"Failed to match author [%s] in database" %
author)
else:
# author exists, check if this book by this author is in our database
# metadata might have quotes in book name
book = book.replace('"', '').replace("'", "")
bookid = find_book_in_db(myDB, author, book)
if bookid:
# check if book is already marked as "Open" (if so,
# we already had it)
check_status = myDB.action(
'SELECT Status from books where BookID="%s"' %
bookid).fetchone()
if check_status['Status'] != 'Open':
# update status as we've got this book
myDB.action(
'UPDATE books set Status="Open" where BookID="%s"' %
bookid)
book_filename = os.path.join(r, files)
# update book location so we can check if it
# gets removed, or allow click-to-open
myDB.action(
'UPDATE books set BookFile="%s" where BookID="%s"' %
(book_filename, bookid))
# update cover file to cover.jpg in book folder (if exists)
bookdir = book_filename.rsplit(os.sep, 1)[0]
coverimg = os.path.join(bookdir, 'cover.jpg')
cachedir = os.path.join(str(lazylibrarian.PROG_DIR), 'data' + os.sep + 'images' + os.sep + 'cache')
cacheimg = os.path.join(cachedir, bookid + '.jpg')
if os.path.isfile(coverimg):
copyfile(coverimg, cacheimg)
new_book_count += 1
else:
logger.debug(
"Failed to match book [%s] by [%s] in database" %
(book, author))
logger.info("%s new/modified book%s found and added to the database" %
(new_book_count, plural(new_book_count)))
logger.info("%s file%s processed" % (file_count, plural(file_count)))
# show statistics of full library scans
if startdir == lazylibrarian.DESTINATION_DIR:
stats = myDB.action(
"SELECT sum(GR_book_hits), sum(GR_lang_hits), sum(LT_lang_hits), sum(GB_lang_change), \
sum(cache_hits), sum(bad_lang), sum(bad_char), sum(uncached), sum(duplicates) FROM stats").fetchone()
if stats['sum(GR_book_hits)'] is not None:
# only show stats if new books added
if lazylibrarian.BOOK_API == "GoogleBooks":
logger.debug("GoogleBooks was hit %s time%s for books" %
(stats['sum(GR_book_hits)'], plural(stats['sum(GR_book_hits)'])))
logger.debug("GoogleBooks language was changed %s time%s" %
(stats['sum(GB_lang_change)'], plural(stats['sum(GB_lang_change)'])))
if lazylibrarian.BOOK_API == "GoodReads":
logger.debug("GoodReads was hit %s time%s for books" %
(stats['sum(GR_book_hits)'], plural(stats['sum(GR_book_hits)'])))
logger.debug("GoodReads was hit %s time%s for languages" %
(stats['sum(GR_lang_hits)'], plural(stats['sum(GR_lang_hits)'])))
logger.debug("LibraryThing was hit %s time%s for languages" %
(stats['sum(LT_lang_hits)'], plural (stats['sum(LT_lang_hits)'])))
logger.debug("Language cache was hit %s time%s" %
(stats['sum(cache_hits)'], plural(stats['sum(cache_hits)'])))
logger.debug("Unwanted language removed %s book%s" %
(stats['sum(bad_lang)'], plural (stats['sum(bad_lang)'])))
logger.debug("Unwanted characters removed %s book%s" %
(stats['sum(bad_char)'], plural(stats['sum(bad_char)'])))
logger.debug("Unable to cache %s book%s with missing ISBN" %
(stats['sum(uncached)'], plural(stats['sum(uncached)'])))
logger.debug("Found %s duplicate book%s" %
(stats['sum(duplicates)'], plural(stats['sum(duplicates)'])))
logger.debug("Cache %s hit%s, %s miss" %
(lazylibrarian.CACHE_HIT, plural(lazylibrarian.CACHE_HIT), lazylibrarian.CACHE_MISS))
cachesize = myDB.action("select count('ISBN') as counter from languages").fetchone()
logger.debug("ISBN Language cache holds %s entries" % cachesize['counter'])
nolang = len(myDB.select('select BookID from Books where status="Open" and BookLang="Unknown"'))
if nolang:
logger.warn("Found %s book%s in your library with unknown language" % (nolang, plural(nolang)))
authors = myDB.select('select AuthorID from authors')
# Update bookcounts for all authors, not just new ones - refresh may have located
# new books for existing authors especially if switched provider gb/gr
else:
# single author/book import
authors = myDB.select('select AuthorID from authors where AuthorName = "%s"' % author)
logger.debug('Updating bookcounts for %i author%s' % (len(authors), plural(len(authors))))
for author in authors:
update_totals(author['AuthorID'])
images = myDB.select('select bookid, bookimg, bookname from books where bookimg like "http%"')
if len(images):
logger.info("Caching cover%s for %i book%s" % (plural(len(images)), len(images), plural(len(images))))
for item in images:
bookid = item['bookid']
bookimg = item['bookimg']
bookname = item['bookname']
newimg = cache_cover(bookid, bookimg)
if newimg is not None:
myDB.action('update books set BookImg="%s" where BookID="%s"' % (newimg, bookid))
images = myDB.select('select AuthorID, AuthorImg, AuthorName from authors where AuthorImg like "http%"')
if len(images):
logger.info("Caching image%s for %i author%s" % (plural(len(images)), len(images), plural(len(images))))
for item in images:
authorid = item['authorid']
authorimg = item['authorimg']
authorname = item['authorname']
newimg = cache_cover(authorid, authorimg)
if newimg is not None:
myDB.action('update authors set AuthorImg="%s" where AuthorID="%s"' % (newimg, authorid))
setWorkPages()
logger.info('Library scan complete')
return new_book_count
def getBookCover(bookID=None):
""" Return link to a local file containing a book cover image for a bookid.
Try 1. Local file cached from goodreads/googlebooks when book was imported
2. cover.jpg if we have the book
3. LibraryThing whatwork
4. Goodreads search if book was imported from goodreads
5. Google images search
Return None if no cover available. """
if not bookID:
logger.error("getBookCover- No bookID")
return None
cachedir = lazylibrarian.CACHEDIR
coverfile = os.path.join(cachedir, bookID + '.jpg')
if os.path.isfile(coverfile): # use cached image if there is one
lazylibrarian.CACHE_HIT = int(lazylibrarian.CACHE_HIT) + 1
logger.debug(u"getBookCover: Returning Cached response for %s" % coverfile)
coverlink = 'cache/' + bookID + '.jpg'
return coverlink
lazylibrarian.CACHE_MISS = int(lazylibrarian.CACHE_MISS) + 1
myDB = database.DBConnection()
item = myDB.match('select BookFile from books where bookID="%s"' % bookID)
if item:
bookfile = item['BookFile']
if bookfile: # we may have a cover.jpg in the same folder
bookdir = os.path.dirname(bookfile)
coverimg = os.path.join(bookdir, "cover.jpg")
if os.path.isfile(coverimg):
logger.debug(u"getBookCover: Copying book cover to %s" % coverfile)
shutil.copyfile(coverimg, coverfile)
coverlink = 'cache/' + bookID + '.jpg'
return coverlink
# if no cover.jpg, see if librarything workpage has a cover
work = getBookWork(bookID, "Cover")
if work:
try:
img = work.split('og:image')[1].split('="')[1].split('"')[0]
if img and img.startswith('http'):
coverlink = cache_cover(bookID, img)
if coverlink:
logger.debug(u"getBookCover: Caching librarything cover for %s" % bookID)
return coverlink
else:
logger.debug("getBookCover: No image found in work page for %s" % bookID)
except IndexError:
logger.debug('getBookCover: Image not found in work page for %s' % bookID)
# not found in librarything work page, try to get a cover from goodreads or google instead
item = myDB.match('select BookName,AuthorName,BookLink from books where bookID="%s"' % bookID)
if item:
title = safe_unicode(item['BookName']).encode(lazylibrarian.SYS_ENCODING)
author = safe_unicode(item['AuthorName']).encode(lazylibrarian.SYS_ENCODING)
booklink = item['BookLink']
safeparams = urllib.quote_plus("%s %s" % (author, title))
if 'goodreads' in booklink:
# if the bookID is a goodreads one, we can call https://www.goodreads.com/book/show/{bookID}
# and scrape the page for og:image
# <meta property="og:image" content="https://i.gr-assets.com/images/S/photo.goodreads.com/books/1388267702i/16304._UY475_SS475_.jpg"/>
# to get the cover
time_now = int(time.time())
if time_now <= lazylibrarian.LAST_GOODREADS:
time.sleep(1)
lazylibrarian.LAST_GOODREADS = time_now
result, success = fetchURL(booklink)
if success:
try:
img = result.split('og:image')[1].split('="')[1].split('"')[0]
except IndexError:
img = None
if img and img.startswith('http') and 'nocover' not in img and 'nophoto' not in img:
time_now = int(time.time())
if time_now <= lazylibrarian.LAST_GOODREADS:
time.sleep(1)
lazylibrarian.LAST_GOODREADS = time_now
coverlink = cache_cover(bookID, img)
if coverlink:
logger.debug("getBookCover: Caching goodreads cover for %s %s" % (author, title))
return coverlink
else:
logger.debug("getBookCover: Error getting goodreads image for %s, [%s]" % (img, result))
else:
logger.debug("getBookCover: No image found in goodreads page for %s" % bookID)
else:
logger.debug("getBookCover: Error getting page %s, [%s]" % (booklink, result))
# if this failed, try a google image search...
# tbm=isch search images
# tbs=isz:l large images
# ift:jpg jpeg file type
URL = "https://www.google.com/search?tbm=isch&tbs=isz:l,ift:jpg&as_q=" + safeparams + "+ebook"
result, success = fetchURL(URL)
if success:
try:
img = result.split('url?q=')[1].split('">')[1].split('src="')[1].split('"')[0]
except IndexError:
img = None
if img and img.startswith('http'):
coverlink = cache_cover(bookID, img)
if coverlink:
logger.debug("getBookCover: Caching google cover for %s %s" % (author, title))
return coverlink
else:
logger.debug("getBookCover: Error getting google image %s, [%s]" % (img, result))
else:
logger.debug("getBookCover: No image found in google page for %s" % bookID)
else:
logger.debug("getBookCover: Error getting google page for %s, [%s]" % (safeparams, result))
return None
def find_book(self, bookid=None, queue=None):
myDB = database.DBConnection()
URL = 'https://www.goodreads.com/book/show/' + bookid + '?' + urllib.urlencode(self.params)
try:
rootxml, in_cache = get_xml_request(URL)
if rootxml is None:
logger.debug("Error requesting book")
return
except Exception as e:
logger.error("Error finding book: %s" % e)
return
bookLanguage = rootxml.find('./book/language_code').text
bookname = rootxml.find('./book/title').text
if not bookLanguage:
bookLanguage = "Unknown"
#
# PAB user has said they want this book, don't block for bad language, just warn
#
valid_langs = ([valid_lang.strip() for valid_lang in lazylibrarian.IMP_PREFLANG.split(',')])
if bookLanguage not in valid_langs:
logger.debug('Book %s language does not match preference' % bookname)
if (rootxml.find('./book/publication_year').text is None):
bookdate = "0000"
else:
bookdate = rootxml.find('./book/publication_year').text
try:
bookimg = rootxml.find('./book/img_url').text
if 'assets/nocover' in bookimg:
bookimg = 'images/nocover.png'
except (KeyError, AttributeError):
bookimg = 'images/nocover.png'
authorname = rootxml.find('./book/authors/author/name').text
bookdesc = rootxml.find('./book/description').text
bookisbn = rootxml.find('./book/isbn').text
bookpub = rootxml.find('./book/publisher').text
booklink = rootxml.find('./book/link').text
bookrate = float(rootxml.find('./book/average_rating').text)
bookpages = rootxml.find('.book/num_pages').text
name = authorname
GR = GoodReads(name)
author = GR.find_author_id()
if author:
AuthorID = author['authorid']
booksub = ''
bookname = unaccented(bookname)
if ': ' in bookname:
parts = bookname.split(': ', 1)
bookname = parts[0]
booksub = parts[1]
dic = {':': '', '"': '', '\'': ''}
bookname = replace_all(bookname, dic)
bookname = bookname.strip() # strip whitespace
booksub = replace_all(booksub, dic)
booksub = booksub.strip() # strip whitespace
if booksub:
series,seriesNum = bookSeries(booksub)
else:
series,seriesNum = bookSeries(bookname)
controlValueDict = {"BookID": bookid}
newValueDict = {
"AuthorName": authorname,
"AuthorID": AuthorID,
"AuthorLink": None,
"BookName": bookname,
"BookSub": booksub,
"BookDesc": bookdesc,
"BookIsbn": bookisbn,
"BookPub": bookpub,
"BookGenre": None,
"BookImg": bookimg,
"BookLink": booklink,
"BookRate": bookrate,
"BookPages": bookpages,
"BookDate": bookdate,
"BookLang": bookLanguage,
"Status": "Wanted",
"BookAdded": today(),
"Series": series,
"SeriesNum": seriesNum
}
myDB.upsert("books", newValueDict, controlValueDict)
logger.debug("%s added to the books database" % bookname)
if 'nocover' in bookimg or 'nophoto' in bookimg:
# try to get a cover from librarything
workcover = getBookCover(bookid)
if workcover:
logger.debug(u'Updated cover for %s to %s' % (bookname, workcover))
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": workcover}
myDB.upsert("books", newValueDict, controlValueDict)
elif bookimg and bookimg.startswith('http'):
link = cache_cover(bookid, bookimg)
if link is not None:
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": link}
myDB.upsert("books", newValueDict, controlValueDict)
if seriesNum == None:
# try to get series info from librarything
series, seriesNum = getWorkSeries(bookid)
if seriesNum:
logger.debug(u'Updated series: %s [%s]' % (series, seriesNum))
controlValueDict = {"BookID": bookid}
newValueDict = {
"Series": series,
"SeriesNum": seriesNum
}
myDB.upsert("books", newValueDict, controlValueDict)
worklink = getWorkPage(bookid)
if worklink:
controlValueDict = {"BookID": bookid}
newValueDict = {"WorkPage": worklink}
myDB.upsert("books", newValueDict, controlValueDict)
def get_author_books(self, authorid=None, authorname=None, refresh=False):
try:
api_hits = 0
gr_lang_hits = 0
lt_lang_hits = 0
gb_lang_change = 0
cache_hits = 0
not_cached = 0
URL = 'http://www.goodreads.com/author/list/' + authorid + '.xml?' + urllib.urlencode(self.params)
# Artist is loading
myDB = database.DBConnection()
controlValueDict = {"AuthorID": authorid}
newValueDict = {"Status": "Loading"}
myDB.upsert("authors", newValueDict, controlValueDict)
try:
rootxml, in_cache = get_xml_request(URL, useCache=not refresh)
except Exception as e:
logger.error("Error fetching author books: %s" % str(e))
return
if rootxml is None:
logger.debug("Error requesting author books")
return
if not in_cache:
api_hits = api_hits + 1
resultxml = rootxml.getiterator('book')
valid_langs = ([valid_lang.strip() for valid_lang in lazylibrarian.IMP_PREFLANG.split(',')])
if not len(resultxml):
logger.warn('[%s] No books found for author with ID: %s' % (authorname, authorid))
else:
logger.debug("[%s] Now processing books with GoodReads API" % authorname)
resultsCount = 0
removedResults = 0
duplicates = 0
ignored = 0
added_count = 0
updated_count = 0
book_ignore_count = 0
total_count = 0
logger.debug(u"url " + URL)
authorNameResult = rootxml.find('./author/name').text
logger.debug(u"author name " + authorNameResult)
loopCount = 1
isbn_979_dict = {
"10": "fre",
"11": "kor",
"12": "ita"
}
isbn_978_dict = {
"0": "eng",
"1": "eng",
"2": "fre",
"3": "ger",
"4": "jap",
"5": "rus"
}
while resultxml:
for book in resultxml:
total_count = total_count + 1
if (book.find('publication_year').text is None):
pubyear = "0000"
else:
pubyear = book.find('publication_year').text
try:
bookimg = book.find('image_url').text
if ('nocover' in bookimg):
bookimg = 'images/nocover.png'
except (KeyError, AttributeError):
bookimg = 'images/nocover.png'
bookLanguage = "Unknown"
find_field = "id"
isbn = ""
isbnhead = ""
if "All" not in valid_langs: # do we care about language
if book.find('isbn').text:
find_field = "isbn"
isbn = book.find('isbn').text
isbnhead = isbn[0:3]
else:
if book.find('isbn13').text:
find_field = "isbn13"
isbn = book.find('isbn13').text
isbnhead = isbn[3:6]
if (find_field != 'id'): # isbn10 or isbn13 found
# Try to use shortcut of ISBN identifier codes described here...
# https://en.wikipedia.org/wiki/List_of_ISBN_identifier_groups
if isbnhead != "":
if find_field == "isbn13" and isbn.startswith('979'):
for item in isbn_979_dict:
if isbnhead.startswith(item):
bookLanguage = isbn_979_dict[item]
break
if bookLanguage != "Unknown":
logger.debug("ISBN979 returned %s for %s" % (bookLanguage, isbnhead))
elif (find_field == "isbn") or (find_field == "isbn13" and isbn.startswith('978')):
for item in isbn_978_dict:
if isbnhead.startswith(item):
bookLanguage = isbn_978_dict[item]
break
if bookLanguage != "Unknown":
logger.debug("ISBN978 returned %s for %s" % (bookLanguage, isbnhead))
if bookLanguage == "Unknown":
# Nothing in the isbn dictionary, try any cached results
match = myDB.match('SELECT lang FROM languages where isbn = "%s"' % (isbnhead))
if match:
bookLanguage = match['lang']
cache_hits = cache_hits + 1
logger.debug("Found cached language [%s] for %s [%s]" %
(bookLanguage, find_field, isbnhead))
else:
# no match in cache, try searching librarything for a language code using the isbn
# if no language found, librarything return value is "invalid" or "unknown"
# returns plain text, not xml
BOOK_URL = 'http://www.librarything.com/api/thingLang.php?isbn=' + isbn
try:
librarything_wait()
resp = urllib2.urlopen(BOOK_URL, timeout=30).read()
lt_lang_hits = lt_lang_hits + 1
logger.debug("LibraryThing reports language [%s] for %s" % (resp, isbnhead))
if ('invalid' in resp or 'Unknown' in resp):
bookLanguage = "Unknown"
else:
bookLanguage = resp # found a language code
myDB.action('insert into languages values ("%s", "%s")' %
(isbnhead, bookLanguage))
logger.debug(u"LT language %s: %s" % (isbnhead, bookLanguage))
except Exception as e:
logger.error("Error finding LT language result for [%s], %s" % (isbn, str(e)))
if bookLanguage == "Unknown":
# still no earlier match, we'll have to search the goodreads api
try:
if book.find(find_field).text:
BOOK_URL = 'http://www.goodreads.com/book/show?id=' + \
book.find(find_field).text + '&' + urllib.urlencode(self.params)
logger.debug(u"Book URL: " + BOOK_URL)
try:
time_now = int(time.time())
if time_now <= lazylibrarian.LAST_GOODREADS:
time.sleep(1)
BOOK_rootxml, in_cache = get_xml_request(BOOK_URL)
if BOOK_rootxml is None:
logger.debug('Error requesting book language code')
bookLanguage = ""
else:
if not in_cache:
# only update last_goodreads if the result wasn't found in the cache
lazylibrarian.LAST_GOODREADS = time_now
bookLanguage = BOOK_rootxml.find('./book/language_code').text
except Exception as e:
logger.error("Error finding book results: %s" % str(e))
if not in_cache:
gr_lang_hits = gr_lang_hits + 1
if not bookLanguage:
bookLanguage = "Unknown"
# At this point, give up?
# WhatWork on author/title doesn't give us a language.
# It might give us the "original language" of the book (but not always)
# and our copy might not be in the original language anyway
# eg "The Girl With the Dragon Tattoo" original language Swedish
# If we have an isbn, try WhatISBN to get alternatives
# in case any of them give us a language, but it seems if thinglang doesn't
# have a language for the first isbn code, it doesn't for any of the
# alternatives either
# Goodreads search results don't include the language. Although sometimes
# it's in the html page, it's not in the xml results
if (isbnhead != ""):
# if GR didn't give an isbn we can't cache it, just use language for this book
myDB.action('insert into languages values ("%s", "%s")' %
(isbnhead, bookLanguage))
logger.debug("GoodReads reports language [%s] for %s" %
(bookLanguage, isbnhead))
else:
not_cached = not_cached + 1
logger.debug(u"GR language: " + bookLanguage)
else:
logger.debug("No %s provided for [%s]" % (find_field, book.find('title').text))
# continue
except Exception as e:
logger.debug(u"Goodreads language search failed: %s" % str(e))
if bookLanguage not in valid_langs:
logger.debug('Skipped %s with language %s' % (book.find('title').text, bookLanguage))
ignored = ignored + 1
continue
bookname = book.find('title').text
bookid = book.find('id').text
bookdesc = book.find('description').text
bookisbn = book.find('isbn').text
bookpub = book.find('publisher').text
booklink = book.find('link').text
bookrate = float(book.find('average_rating').text)
bookpages = book.find('num_pages').text
bookname = unaccented(bookname)
bookname, booksub = split_title(authorNameResult, bookname)
dic = {':': '', '"': '', '\'': ''}
bookname = replace_all(bookname, dic)
bookname = bookname.strip() # strip whitespace
booksub = replace_all(booksub, dic)
booksub = booksub.strip() # strip whitespace
if booksub:
series, seriesNum = bookSeries(booksub)
else:
series, seriesNum = bookSeries(bookname)
# GoodReads sometimes has multiple bookids for the same book (same author/title, different editions)
# and sometimes uses the same bookid if the book is the same but the title is slightly different
# We use bookid, then reject if another author/title has a different bookid so we just keep one...
find_book_status = myDB.select('SELECT * FROM books WHERE BookID = "%s"' % bookid)
if find_book_status:
for resulted in find_book_status:
book_status = resulted['Status']
locked = resulted['Manual']
else:
book_status = lazylibrarian.NEWBOOK_STATUS
locked = False
rejected = False
if re.match('[^\w-]', bookname): # reject books with bad characters in title
logger.debug(u"removed result [" + bookname + "] for bad characters")
removedResults = removedResults + 1
rejected = True
if not rejected and not bookname:
logger.debug('Rejecting bookid %s for %s, no bookname' %
(bookid, authorNameResult))
removedResults = removedResults + 1
rejected = True
if not rejected:
find_books = myDB.select('SELECT * FROM books WHERE BookName = "%s" and AuthorName = "%s"' %
(bookname, authorNameResult.replace('"', '""')))
if find_books:
for find_book in find_books:
if find_book['BookID'] != bookid:
# we have a book with this author/title already
logger.debug('Rejecting bookid %s for [%s][%s] already got %s' %
(find_book['BookID'], authorNameResult, bookname, bookid))
duplicates = duplicates + 1
rejected = True
if not rejected:
find_books = myDB.match('SELECT AuthorName,BookName FROM books WHERE BookID = "%s"' % bookid)
if find_books:
# we have a book with this bookid already
if bookname != find_books['BookName'] or authorNameResult != find_books['AuthorName']:
logger.debug('Rejecting bookid %s for [%s][%s] already got bookid for [%s][%s]' %
(bookid, authorNameResult, bookname,
find_books['AuthorName'], find_books['BookName']))
else:
logger.debug('Rejecting bookid %s for [%s][%s] already got this book in database' %
(bookid, authorNameResult, bookname))
duplicates = duplicates + 1
rejected = True
if not rejected:
if book_status != "Ignored":
if not locked:
controlValueDict = {"BookID": bookid}
newValueDict = {
"AuthorName": authorNameResult,
"AuthorID": authorid,
"AuthorLink": None,
"BookName": bookname,
"BookSub": booksub,
"BookDesc": bookdesc,
"BookIsbn": bookisbn,
"BookPub": bookpub,
"BookGenre": None,
"BookImg": bookimg,
"BookLink": booklink,
"BookRate": bookrate,
"BookPages": bookpages,
"BookDate": pubyear,
"BookLang": bookLanguage,
"Status": book_status,
"BookAdded": today(),
"Series": series,
"SeriesNum": seriesNum
}
resultsCount = resultsCount + 1
myDB.upsert("books", newValueDict, controlValueDict)
logger.debug(u"Book found: " + book.find('title').text + " " + pubyear)
if 'nocover' in bookimg or 'nophoto' in bookimg:
# try to get a cover from librarything
workcover = getBookCover(bookid)
if workcover:
logger.debug(u'Updated cover for %s to %s' % (bookname, workcover))
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": workcover}
myDB.upsert("books", newValueDict, controlValueDict)
elif bookimg and bookimg.startswith('http'):
link = cache_cover(bookid, bookimg)
if link:
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": link}
myDB.upsert("books", newValueDict, controlValueDict)
if seriesNum is None:
# try to get series info from librarything
series, seriesNum = getWorkSeries(bookid)
if seriesNum:
logger.debug(u'Updated series: %s [%s]' % (series, seriesNum))
controlValueDict = {"BookID": bookid}
newValueDict = {
"Series": series,
"SeriesNum": seriesNum
}
myDB.upsert("books", newValueDict, controlValueDict)
worklink = getWorkPage(bookid)
if worklink:
controlValueDict = {"BookID": bookid}
newValueDict = {"WorkPage": worklink}
myDB.upsert("books", newValueDict, controlValueDict)
if not find_book_status:
logger.debug(u"[%s] Added book: %s" % (authorname, bookname))
added_count = added_count + 1
else:
logger.debug(u"[%s] Updated book: %s" % (authorname, bookname))
updated_count = updated_count + 1
else:
book_ignore_count = book_ignore_count + 1
loopCount = loopCount + 1
URL = 'http://www.goodreads.com/author/list/' + authorid + '.xml?' + \
urllib.urlencode(self.params) + '&page=' + str(loopCount)
resultxml = None
try:
rootxml, in_cache = get_xml_request(URL, useCache=not refresh)
if rootxml is None:
logger.debug('Error requesting next page of results')
else:
resultxml = rootxml.getiterator('book')
if not in_cache:
api_hits = api_hits + 1
except Exception as e:
resultxml = None
logger.error("Error finding next page of results: %s" % str(e))
if resultxml:
if all(False for book in resultxml): # returns True if iterator is empty
resultxml = None
lastbook = myDB.match('SELECT BookName, BookLink, BookDate from books WHERE AuthorID="%s" \
AND Status != "Ignored" order by BookDate DESC' % authorid)
if lastbook:
lastbookname = lastbook['BookName']
lastbooklink = lastbook['BookLink']
lastbookdate = lastbook['BookDate']
else:
lastbookname = None
lastbooklink = None
lastbookdate = None
controlValueDict = {"AuthorID": authorid}
newValueDict = {
"Status": "Active",
"LastBook": lastbookname,
"LastLink": lastbooklink,
"LastDate": lastbookdate
}
myDB.upsert("authors", newValueDict, controlValueDict)
# This is here because GoodReads sometimes has several entries with the same BookID!
modified_count = added_count + updated_count
logger.debug("Found %s total book%s for author" % (total_count, plural(total_count)))
logger.debug("Removed %s unwanted language result%s for author" % (ignored, plural(ignored)))
logger.debug(
"Removed %s bad character or no-name result%s for author" %
(removedResults, plural(removedResults)))
logger.debug("Removed %s duplicate result%s for author" % (duplicates, plural(duplicates)))
logger.debug("Found %s book%s by author marked as Ignored" % (book_ignore_count, plural(book_ignore_count)))
logger.debug("Imported/Updated %s book%s for author" % (modified_count, plural(modified_count)))
myDB.action('insert into stats values ("%s", %i, %i, %i, %i, %i, %i, %i, %i, %i)' %
(authorname.replace('"', '""'), api_hits, gr_lang_hits, lt_lang_hits, gb_lang_change,
cache_hits, ignored, removedResults, not_cached, duplicates))
if refresh:
logger.info("[%s] Book processing complete: Added %s book%s / Updated %s book%s" %
(authorname, added_count, plural(added_count), updated_count, plural(updated_count)))
else:
logger.info("[%s] Book processing complete: Added %s book%s to the database" %
(authorname, added_count, plural(added_count)))
except Exception as e:
logger.error('Unhandled exception in GR.get_author_books: %s' % traceback.format_exc())
