def _Magazine(self, **kwargs):
index = 0
if 'index' in kwargs:
index = check_int(kwargs['index'], 0)
myDB = database.DBConnection()
if 'magid' not in kwargs:
self.data = self._error_with_message('No Magazine Provided')
return
links = []
entries = []
title = ''
cmd = "SELECT Title,IssueID,IssueDate,IssueAcquired,IssueFile from issues "
cmd += "WHERE Title='%s' order by IssueDate DESC"
results = myDB.select(cmd % kwargs['magid'])
page = results[index:(index + self.PAGE_SIZE)]
for issue in page:
title = makeUnicode(issue['Title'])
entry = {'title': escape('%s (%s)' % (title, issue['IssueDate'])),
'id': escape('issue:%s' % issue['IssueID']),
'updated': opdstime(issue['IssueAcquired']),
'content': escape('%s - %s' % (title, issue['IssueDate'])),
'href': '%s?cmd=Serve&issueid=%s' % (self.opdsroot, quote_plus(issue['IssueID'])),
'kind': 'acquisition',
'rel': 'file',
'type': mimeType(issue['IssueFile'])}
if lazylibrarian.CONFIG['OPDS_METAINFO']:
fname = os.path.splitext(issue['IssueFile'])[0]
res = cache_img('magazine', issue['IssueID'], fname + '.jpg')
entry['image'] = self.searchroot + '/' + res[0]
entries.append(entry)
feed = {}
title = '%s (%s)' % (escape(title), len(entries))
feed['title'] = 'LazyLibrarian OPDS - %s' % title
feed['id'] = 'magazine:%s' % escape(kwargs['magid'])
feed['updated'] = now()
links.append(getLink(href=self.opdsroot, ftype='application/atom+xml; profile=opds-catalog; kind=navigation',
rel='start', title='Home'))
links.append(getLink(href='%s?cmd=Magazine&magid=%s' % (self.opdsroot, quote_plus(kwargs['magid'])),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='self'))
if len(results) > (index + self.PAGE_SIZE):
links.append(
getLink(href='%s?cmd=Magazine&magid=%s&index=%s' % (self.opdsroot, quote_plus(kwargs['magid']),
index + self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='next'))
if index >= self.PAGE_SIZE:
links.append(
getLink(href='%s?cmd=Magazine&magid=%s&index=%s' % (self.opdsroot, quote_plus(kwargs['magid']),
index - self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='previous'))
feed['links'] = links
feed['entries'] = entries
logger.debug("Returning %s issue%s" % (len(entries), plural(len(entries))))
self.data = feed
return
def getAuthorImage(authorid=None):
# tbm=isch search images
# tbs=ift:jpg jpeg file type
if not authorid:
logger.error("getAuthorImage: No authorid")
return None
cachedir = lazylibrarian.CACHEDIR
coverfile = os.path.join(cachedir, "author", authorid + '.jpg')
if os.path.isfile(coverfile): # use cached image if there is one
lazylibrarian.CACHE_HIT = int(lazylibrarian.CACHE_HIT) + 1
logger.debug("getAuthorImage: Returning Cached response for %s" %
coverfile)
coverlink = 'cache/author/' + authorid + '.jpg'
return coverlink
lazylibrarian.CACHE_MISS = int(lazylibrarian.CACHE_MISS) + 1
myDB = database.DBConnection()
author = myDB.match('select AuthorName from authors where AuthorID=?',
(authorid, ))
if author:
authorname = safe_unicode(author['AuthorName'])
if PY2:
authorname = authorname.encode(lazylibrarian.SYS_ENCODING)
safeparams = quote_plus("author %s" % authorname)
URL = "https://www.google.com/search?tbm=isch&tbs=ift:jpg,itp:face&as_q=" + safeparams + 'author'
result, success = fetchURL(URL)
if success:
try:
img = result.split('url?q=')[1].split('">')[1].split(
'src="')[1].split('"')[0]
except IndexError:
img = None
if img and img.startswith('http'):
coverlink, success, was_in_cache = cache_img(
"author", authorid, img)
if success:
if was_in_cache:
logger.debug("Returning cached google image for %s" %
authorname)
else:
logger.debug("Cached google image for %s" % authorname)
return coverlink
else:
logger.debug("Error getting google image %s, [%s]" %
(img, coverlink))
else:
logger.debug("No image found in google page for %s" %
authorname)
else:
logger.debug("Error getting google page for %s, [%s]" %
(safeparams, result))
else:
logger.debug("No author found for %s" % authorid)
return None
def _Magazine(self, **kwargs):
index = 0
if 'index' in kwargs:
index = check_int(kwargs['index'], 0)
myDB = database.DBConnection()
if 'magid' not in kwargs:
self.data = self._error_with_message('No Magazine Provided')
return
links = []
entries = []
title = ''
cmd = "SELECT Title,IssueID,IssueDate,IssueAcquired,IssueFile from issues "
cmd += "WHERE Title='%s' order by IssueDate DESC"
results = myDB.select(cmd % kwargs['magid'])
page = results[index:(index + self.PAGE_SIZE)]
for issue in page:
title = makeUnicode(issue['Title'])
entry = {'title': escape('%s (%s)' % (title, issue['IssueDate'])),
'id': escape('issue:%s' % issue['IssueID']),
'updated': opdstime(issue['IssueAcquired']),
'content': escape('%s - %s' % (title, issue['IssueDate'])),
'href': '%s?cmd=Serve&issueid=%s' % (self.opdsroot, quote_plus(issue['IssueID'])),
'kind': 'acquisition',
'rel': 'file',
'type': mimeType(issue['IssueFile'])}
if lazylibrarian.CONFIG['OPDS_METAINFO']:
fname = os.path.splitext(issue['IssueFile'])[0]
res = cache_img('magazine', issue['IssueID'], fname + '.jpg')
entry['image'] = self.searchroot + '/' + res[0]
entries.append(entry)
feed = {}
title = '%s (%s)' % (escape(title), len(entries))
feed['title'] = 'LazyLibrarian OPDS - %s' % title
feed['id'] = 'magazine:%s' % escape(kwargs['magid'])
feed['updated'] = now()
links.append(getLink(href=self.opdsroot, ftype='application/atom+xml; profile=opds-catalog; kind=navigation',
rel='start', title='Home'))
links.append(getLink(href='%s?cmd=Magazine&magid=%s' % (self.opdsroot, quote_plus(kwargs['magid'])),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='self'))
if len(results) > (index + self.PAGE_SIZE):
links.append(
getLink(href='%s?cmd=Magazine&magid=%s&index=%s' % (self.opdsroot, quote_plus(kwargs['magid']),
index + self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='next'))
if index >= self.PAGE_SIZE:
links.append(
getLink(href='%s?cmd=Magazine&magid=%s&index=%s' % (self.opdsroot, quote_plus(kwargs['magid']),
index - self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='previous'))
feed['links'] = links
feed['entries'] = entries
logger.debug("Returning %s issue%s" % (len(entries), plural(len(entries))))
self.data = feed
return
def _RecentMags(self, **kwargs):
index = 0
if 'index' in kwargs:
index = check_int(kwargs['index'], 0)
myDB = database.DBConnection()
feed = {'title': 'LazyLibrarian OPDS - Recent Magazines', 'id': 'Recent Magazines', 'updated': now()}
links = []
entries = []
links.append(getLink(href=self.opdsroot, ftype='application/atom+xml; profile=opds-catalog; kind=navigation',
rel='start', title='Home'))
links.append(getLink(href='%s?cmd=RecentMags' % self.opdsroot,
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='self'))
links.append(getLink(href='%s/opensearchmagazines.xml' % self.searchroot,
ftype='application/opensearchdescription+xml', rel='search', title='Search Magazines'))
cmd = "select Title,IssueID,IssueAcquired,IssueDate,IssueFile from issues "
cmd += "where IssueFile != '' "
if 'query' in kwargs:
cmd += "AND Title LIKE '%" + kwargs['query'] + "%' "
cmd += "order by IssueAcquired DESC"
results = myDB.select(cmd)
page = results[index:(index + self.PAGE_SIZE)]
for mag in page:
title = makeUnicode(mag['Title'])
entry = {'title': escape('%s' % mag['IssueDate']),
'id': escape('issue:%s' % mag['IssueID']),
'updated': opdstime(mag['IssueAcquired']),
'content': escape('%s - %s' % (title, mag['IssueDate'])),
'href': '%s?cmd=Serve&issueid=%s' % (self.opdsroot, quote_plus(mag['IssueID'])),
'kind': 'acquisition',
'rel': 'file',
'author': escape(title),
'type': mimeType(mag['IssueFile'])}
if lazylibrarian.CONFIG['OPDS_METAINFO']:
fname = os.path.splitext(mag['IssueFile'])[0]
res = cache_img('magazine', mag['IssueID'], fname + '.jpg')
entry['image'] = self.searchroot + '/' + res[0]
entries.append(entry)
if len(results) > (index + self.PAGE_SIZE):
links.append(
getLink(href='%s?cmd=RecentMags&index=%s' % (self.opdsroot, index + self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='next'))
if index >= self.PAGE_SIZE:
links.append(
getLink(href='%s?cmd=RecentMags&index=%s' % (self.opdsroot, index - self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='previous'))
feed['links'] = links
feed['entries'] = entries
logger.debug("Returning %s issue%s" % (len(entries), plural(len(entries))))
self.data = feed
return
def _Magazines(self, **kwargs):
index = 0
if 'index' in kwargs:
index = check_int(kwargs['index'], 0)
myDB = database.DBConnection()
feed = {'title': 'LazyLibrarian OPDS - Magazines', 'id': 'Magazines', 'updated': now()}
links = []
entries = []
links.append(getLink(href=self.opdsroot, ftype='application/atom+xml; profile=opds-catalog; kind=navigation',
rel='start', title='Home'))
links.append(getLink(href='%s?cmd=Magazines' % self.opdsroot,
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='self'))
links.append(getLink(href='%s/opensearchmagazines.xml' % self.searchroot,
ftype='application/opensearchdescription+xml', rel='search', title='Search Magazines'))
cmd = 'select magazines.*,(select count(*) as counter from issues where magazines.title = issues.title)'
cmd += ' as Iss_Cnt from magazines '
if 'query' in kwargs:
cmd += "WHERE magazines.title LIKE '%" + kwargs['query'] + "%' "
cmd += 'order by magazines.title'
results = myDB.select(cmd)
page = results[index:(index + self.PAGE_SIZE)]
for mag in page:
if mag['Iss_Cnt'] > 0:
title = makeUnicode(mag['Title'])
entry = {
'title': escape('%s (%s)' % (title, mag['Iss_Cnt'])),
'id': escape('magazine:%s' % title),
'updated': opdstime(mag['LastAcquired']),
'content': escape('%s' % title),
'href': '%s?cmd=Magazine&magid=%s' % (self.opdsroot, quote_plus(title)),
'kind': 'navigation',
'rel': 'subsection',
}
if lazylibrarian.CONFIG['OPDS_METAINFO']:
res = cache_img('magazine', md5_utf8(mag['LatestCover']), mag['LatestCover'], refresh=True)
entry['image'] = self.searchroot + '/' + res[0]
entries.append(entry)
if len(results) > (index + self.PAGE_SIZE):
links.append(
getLink(href='%s?cmd=Magazines&index=%s' % (self.opdsroot, index + self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='next'))
if index >= self.PAGE_SIZE:
links.append(
getLink(href='%s?cmd=Magazines&index=%s' % (self.opdsroot, index - self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='previous'))
feed['links'] = links
feed['entries'] = entries
logger.debug("Returning %s magazine%s" % (len(entries), plural(len(entries))))
self.data = feed
return
def _RecentMags(self, **kwargs):
index = 0
if 'index' in kwargs:
index = check_int(kwargs['index'], 0)
myDB = database.DBConnection()
feed = {'title': 'LazyLibrarian OPDS - Recent Magazines', 'id': 'Recent Magazines', 'updated': now()}
links = []
entries = []
links.append(getLink(href=self.opdsroot, ftype='application/atom+xml; profile=opds-catalog; kind=navigation',
rel='start', title='Home'))
links.append(getLink(href='%s?cmd=RecentMags' % self.opdsroot,
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='self'))
links.append(getLink(href='%s/opensearchmagazines.xml' % self.searchroot,
ftype='application/opensearchdescription+xml', rel='search', title='Search Magazines'))
cmd = "select Title,IssueID,IssueAcquired,IssueDate,IssueFile from issues "
cmd += "where IssueFile != '' "
if 'query' in kwargs:
cmd += "AND Title LIKE '%" + kwargs['query'] + "%' "
cmd += "order by IssueAcquired DESC"
results = myDB.select(cmd)
page = results[index:(index + self.PAGE_SIZE)]
for mag in page:
title = makeUnicode(mag['Title'])
entry = {'title': escape('%s' % mag['IssueDate']),
'id': escape('issue:%s' % mag['IssueID']),
'updated': opdstime(mag['IssueAcquired']),
'content': escape('%s - %s' % (title, mag['IssueDate'])),
'href': '%s?cmd=Serve&issueid=%s' % (self.opdsroot, quote_plus(mag['IssueID'])),
'kind': 'acquisition',
'rel': 'file',
'author': escape(title),
'type': mimeType(mag['IssueFile'])}
if lazylibrarian.CONFIG['OPDS_METAINFO']:
fname = os.path.splitext(mag['IssueFile'])[0]
res = cache_img('magazine', mag['IssueID'], fname + '.jpg')
entry['image'] = self.searchroot + '/' + res[0]
entries.append(entry)
if len(results) > (index + self.PAGE_SIZE):
links.append(
getLink(href='%s?cmd=RecentMags&index=%s' % (self.opdsroot, index + self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='next'))
if index >= self.PAGE_SIZE:
links.append(
getLink(href='%s?cmd=RecentMags&index=%s' % (self.opdsroot, index - self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='previous'))
feed['links'] = links
feed['entries'] = entries
logger.debug("Returning %s issue%s" % (len(entries), plural(len(entries))))
self.data = feed
return
def getAuthorImage(authorid=None):
# tbm=isch search images
# tbs=ift:jpg jpeg file type
if not authorid:
logger.error("getAuthorImage: No authorid")
return None
cachedir = lazylibrarian.CACHEDIR
coverfile = os.path.join(cachedir, "author", authorid + '.jpg')
if os.path.isfile(coverfile): # use cached image if there is one
lazylibrarian.CACHE_HIT = int(lazylibrarian.CACHE_HIT) + 1
logger.debug("getAuthorImage: Returning Cached response for %s" % coverfile)
coverlink = 'cache/author/' + authorid + '.jpg'
return coverlink
lazylibrarian.CACHE_MISS = int(lazylibrarian.CACHE_MISS) + 1
myDB = database.DBConnection()
author = myDB.match('select AuthorName from authors where AuthorID=?', (authorid,))
if author:
authorname = safe_unicode(author['AuthorName'])
if PY2:
authorname = authorname.encode(lazylibrarian.SYS_ENCODING)
safeparams = quote_plus("author %s" % authorname)
URL = "https://www.google.com/search?tbm=isch&tbs=ift:jpg,itp:face&as_q=" + safeparams + 'author'
result, success = fetchURL(URL)
if success:
try:
img = result.split('url?q=')[1].split('">')[1].split('src="')[1].split('"')[0]
except IndexError:
img = None
if img and img.startswith('http'):
coverlink, success, was_in_cache = cache_img("author", authorid, img)
if success:
if was_in_cache:
logger.debug("Returning cached google image for %s" % authorname)
else:
logger.debug("Cached google image for %s" % authorname)
return coverlink
else:
logger.debug("Error getting google image %s, [%s]" % (img, coverlink))
else:
logger.debug("No image found in google page for %s" % authorname)
else:
logger.debug("Error getting google page for %s, [%s]" % (safeparams, result))
else:
logger.debug("No author found for %s" % authorid)
return None
def _setimage(self, table, itemid, img):
msg = "%s Image [%s] rejected" % (table, img)
# Cache file image
if os.path.isfile(img):
extn = os.path.splitext(img)[1].lower()
if extn and extn in ['.jpg', '.jpeg', '.png']:
destfile = os.path.join(lazylibrarian.CACHEDIR, table,
itemid + '.jpg')
try:
shutil.copy(img, destfile)
setperm(destfile)
msg = ''
except Exception as why:
msg += " Failed to copy file: %s %s" % (type(why).__name__,
str(why))
else:
msg += " invalid extension"
if img.startswith('http'):
# cache image from url
extn = os.path.splitext(img)[1].lower()
if extn and extn in ['.jpg', '.jpeg', '.png']:
cachedimg, success = cache_img(table, itemid, img)
if success:
msg = ''
else:
msg += " Failed to cache file"
else:
msg += " invalid extension"
elif msg:
msg += " Not found"
if msg:
self.data = msg
return
myDB = database.DBConnection()
dbentry = myDB.match('SELECT %sID from %ss WHERE %sID=%s' %
(table, table, table, itemid))
if dbentry:
myDB.action('UPDATE %ss SET %sImg="%s" WHERE %sID=%s' %
(table, table, 'cache' + os.sep + itemid + '.jpg',
table, itemid))
else:
self.data = "%sID %s not found" % (table, itemid)
def find_book(self, bookid=None, bookstatus="None"):
myDB = database.DBConnection()
if not lazylibrarian.CONFIG['GB_API']:
logger.warn('No GoogleBooks API key, check config')
URL = 'https://www.googleapis.com/books/v1/volumes/' + \
str(bookid) + "?key=" + lazylibrarian.CONFIG['GB_API']
jsonresults, in_cache = gb_json_request(URL)
if jsonresults is None:
logger.debug('No results found for %s' % bookid)
return
if not bookstatus:
bookstatus = lazylibrarian.CONFIG['NEWBOOK_STATUS']
book = bookdict(jsonresults)
dic = {':': '.', '"': '', '\'': ''}
bookname = replace_all(book['name'], dic)
bookname = unaccented(bookname)
bookname = bookname.strip() # strip whitespace
if not book['author']:
logger.debug('Book %s does not contain author field, skipping' % bookname)
return
# warn if language is in ignore list, but user said they wanted this book
valid_langs = getList(lazylibrarian.CONFIG['IMP_PREFLANG'])
if book['lang'] not in valid_langs and 'All' not in valid_langs:
logger.debug('Book %s googlebooks language does not match preference, %s' % (bookname, book['lang']))
if lazylibrarian.CONFIG['NO_PUBDATE']:
if not book['date'] or book['date'] == '0000':
logger.warn('Book %s Publication date does not match preference, %s' % (bookname, book['date']))
if lazylibrarian.CONFIG['NO_FUTURE']:
if book['date'] > today()[:4]:
logger.warn('Book %s Future publication date does not match preference, %s' % (bookname, book['date']))
authorname = book['author']
GR = GoodReads(authorname)
author = GR.find_author_id()
if author:
AuthorID = author['authorid']
match = myDB.match('SELECT AuthorID from authors WHERE AuthorID=?', (AuthorID,))
if not match:
match = myDB.match('SELECT AuthorID from authors WHERE AuthorName=?', (author['authorname'],))
if match:
logger.debug('%s: Changing authorid from %s to %s' %
(author['authorname'], AuthorID, match['AuthorID']))
AuthorID = match['AuthorID'] # we have a different authorid for that authorname
else: # no author but request to add book, add author with newauthor status
# User hit "add book" button from a search or a wishlist import
newauthor_status = 'Active'
if lazylibrarian.CONFIG['NEWAUTHOR_STATUS'] in ['Skipped', 'Ignored']:
newauthor_status = 'Paused'
controlValueDict = {"AuthorID": AuthorID}
newValueDict = {
"AuthorName": author['authorname'],
"AuthorImg": author['authorimg'],
"AuthorLink": author['authorlink'],
"AuthorBorn": author['authorborn'],
"AuthorDeath": author['authordeath'],
"DateAdded": today(),
"Status": newauthor_status
}
authorname = author['authorname']
myDB.upsert("authors", newValueDict, controlValueDict)
if lazylibrarian.CONFIG['NEWAUTHOR_BOOKS']:
self.get_author_books(AuthorID, entrystatus=lazylibrarian.CONFIG['NEWAUTHOR_STATUS'])
else:
logger.warn("No AuthorID for %s, unable to add book %s" % (book['author'], bookname))
return
controlValueDict = {"BookID": bookid}
newValueDict = {
"AuthorID": AuthorID,
"BookName": bookname,
"BookSub": book['sub'],
"BookDesc": book['desc'],
"BookIsbn": book['isbn'],
"BookPub": book['pub'],
"BookGenre": book['genre'],
"BookImg": book['img'],
"BookLink": book['link'],
"BookRate": float(book['rate']),
"BookPages": book['pages'],
"BookDate": book['date'],
"BookLang": book['lang'],
"Status": bookstatus,
"AudioStatus": lazylibrarian.CONFIG['NEWAUDIO_STATUS'],
"BookAdded": today()
}
myDB.upsert("books", newValueDict, controlValueDict)
logger.info("%s by %s added to the books database" % (bookname, authorname))
if 'nocover' in book['img'] or 'nophoto' in book['img']:
# try to get a cover from another source
workcover, source = getBookCover(bookid)
if workcover:
logger.debug('Updated cover for %s using %s' % (bookname, source))
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": workcover}
myDB.upsert("books", newValueDict, controlValueDict)
elif book['img'] and book['img'].startswith('http'):
link, success, _ = cache_img("book", bookid, book['img'])
if success:
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": link}
myDB.upsert("books", newValueDict, controlValueDict)
else:
logger.debug('Failed to cache image for %s' % book['img'])
serieslist = []
if book['series']:
serieslist = [('', book['seriesNum'], cleanName(unaccented(book['series']), '&/'))]
if lazylibrarian.CONFIG['ADD_SERIES']:
newserieslist = getWorkSeries(bookid)
if newserieslist:
serieslist = newserieslist
logger.debug('Updated series: %s [%s]' % (bookid, serieslist))
setSeries(serieslist, bookid)
worklink = getWorkPage(bookid)
if worklink:
controlValueDict = {"BookID": bookid}
newValueDict = {"WorkPage": worklink}
myDB.upsert("books", newValueDict, controlValueDict)
def find_book(bookid=None, queue=None):
myDB = database.DBConnection()
if not lazylibrarian.CONFIG['GB_API']:
logger.warn('No GoogleBooks API key, check config')
URL = 'https://www.googleapis.com/books/v1/volumes/' + \
str(bookid) + "?key=" + lazylibrarian.CONFIG['GB_API']
jsonresults, in_cache = get_json_request(URL)
if not jsonresults:
logger.debug('No results found for %s' % bookid)
return
bookname = jsonresults['volumeInfo']['title']
dic = {':': '.', '"': '', '\'': ''}
bookname = replace_all(bookname, dic)
bookname = unaccented(bookname)
bookname = bookname.strip() # strip whitespace
try:
authorname = jsonresults['volumeInfo']['authors'][0]
except KeyError:
logger.debug('Book %s does not contain author field, skipping' %
bookname)
return
try:
# warn if language is in ignore list, but user said they wanted this book
booklang = jsonresults['volumeInfo']['language']
valid_langs = getList(lazylibrarian.CONFIG['IMP_PREFLANG'])
if booklang not in valid_langs and 'All' not in valid_langs:
logger.debug(
'Book %s googlebooks language does not match preference, %s'
% (bookname, booklang))
except KeyError:
logger.debug('Book does not have language field')
booklang = "Unknown"
try:
bookpub = jsonresults['volumeInfo']['publisher']
except KeyError:
bookpub = ""
series = ""
seriesNum = ""
try:
booksub = jsonresults['volumeInfo']['subtitle']
try:
series = booksub.split('(')[1].split(' Series ')[0]
except IndexError:
series = ""
try:
seriesNum = booksub.split('(')[1].split(' Series ')[1].split(
')')[0]
if seriesNum[0] == '#':
seriesNum = seriesNum[1:]
except IndexError:
seriesNum = ""
except KeyError:
booksub = ""
try:
bookdate = jsonresults['volumeInfo']['publishedDate']
except KeyError:
bookdate = '0000-00-00'
try:
bookimg = jsonresults['volumeInfo']['imageLinks']['thumbnail']
except KeyError:
bookimg = 'images/nocover.png'
try:
bookrate = jsonresults['volumeInfo']['averageRating']
except KeyError:
bookrate = 0
try:
bookpages = jsonresults['volumeInfo']['pageCount']
except KeyError:
bookpages = 0
try:
bookgenre = jsonresults['volumeInfo']['categories'][0]
except KeyError:
bookgenre = ""
try:
bookdesc = jsonresults['volumeInfo']['description']
except KeyError:
bookdesc = ""
try:
if jsonresults['volumeInfo']['industryIdentifiers'][0][
'type'] == 'ISBN_10':
bookisbn = jsonresults['volumeInfo']['industryIdentifiers'][0][
'identifier']
else:
bookisbn = ""
except KeyError:
bookisbn = ""
booklink = jsonresults['volumeInfo']['canonicalVolumeLink']
bookrate = float(bookrate)
GR = GoodReads(authorname)
author = GR.find_author_id()
if author:
AuthorID = author['authorid']
match = myDB.match(
'SELECT AuthorID from authors WHERE AuthorID="%s"' % AuthorID)
if not match:
match = myDB.match(
'SELECT AuthorID from authors WHERE AuthorName="%s"' %
author['authorname'])
if match:
logger.debug(
'%s: Changing authorid from %s to %s' %
(author['authorname'], AuthorID, match['AuthorID']))
AuthorID = match[
'AuthorID'] # we have a different authorid for that authorname
else: # no author but request to add book, add author as "ignored"
# User hit "add book" button from a search
controlValueDict = {"AuthorID": AuthorID}
newValueDict = {
"AuthorName": author['authorname'],
"AuthorImg": author['authorimg'],
"AuthorLink": author['authorlink'],
"AuthorBorn": author['authorborn'],
"AuthorDeath": author['authordeath'],
"DateAdded": today(),
"Status": "Ignored"
}
myDB.upsert("authors", newValueDict, controlValueDict)
else:
logger.warn("No AuthorID for %s, unable to add book %s" %
(authorname, bookname))
return
controlValueDict = {"BookID": bookid}
newValueDict = {
"AuthorID": AuthorID,
"BookName": bookname,
"BookSub": booksub,
"BookDesc": bookdesc,
"BookIsbn": bookisbn,
"BookPub": bookpub,
"BookGenre": bookgenre,
"BookImg": bookimg,
"BookLink": booklink,
"BookRate": bookrate,
"BookPages": bookpages,
"BookDate": bookdate,
"BookLang": booklang,
"Status": "Wanted",
"BookAdded": today()
}
myDB.upsert("books", newValueDict, controlValueDict)
logger.info("%s added to the books database" % bookname)
if 'nocover' in bookimg or 'nophoto' in bookimg:
# try to get a cover from librarything
workcover = getBookCover(bookid)
if workcover:
logger.debug(u'Updated cover for %s to %s' %
(bookname, workcover))
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": workcover}
myDB.upsert("books", newValueDict, controlValueDict)
elif bookimg and bookimg.startswith('http'):
link, success = cache_img("book", bookid, bookimg)
if success:
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": link}
myDB.upsert("books", newValueDict, controlValueDict)
else:
logger.debug('Failed to cache image for %s' % bookimg)
if lazylibrarian.CONFIG['ADD_SERIES']:
# prefer series info from librarything
seriesdict = getWorkSeries(bookid)
if seriesdict:
logger.debug(u'Updated series: %s [%s]' % (bookid, seriesdict))
else:
if series:
seriesdict = {cleanName(unaccented(series)): seriesNum}
setSeries(seriesdict, bookid)
worklink = getWorkPage(bookid)
if worklink:
controlValueDict = {"BookID": bookid}
newValueDict = {"WorkPage": worklink}
myDB.upsert("books", newValueDict, controlValueDict)
def get_author_books(self,
authorid=None,
authorname=None,
bookstatus="Skipped",
refresh=False):
try:
logger.debug('[%s] Now processing books with Google Books API' %
authorname)
# google doesnt like accents in author names
set_url = self.url + urllib.quote(
'inauthor:"%s"' % unaccented_str(authorname))
api_hits = 0
gr_lang_hits = 0
lt_lang_hits = 0
gb_lang_change = 0
cache_hits = 0
not_cached = 0
startindex = 0
resultcount = 0
removedResults = 0
duplicates = 0
ignored = 0
added_count = 0
updated_count = 0
book_ignore_count = 0
total_count = 0
number_results = 1
valid_langs = getList(lazylibrarian.CONFIG['IMP_PREFLANG'])
# Artist is loading
myDB = database.DBConnection()
controlValueDict = {"AuthorID": authorid}
newValueDict = {"Status": "Loading"}
myDB.upsert("authors", newValueDict, controlValueDict)
try:
while startindex < number_results:
self.params['startIndex'] = startindex
URL = set_url + '&' + urllib.urlencode(self.params)
try:
jsonresults, in_cache = get_json_request(
URL, useCache=not refresh)
if not jsonresults:
number_results = 0
else:
if not in_cache:
api_hits += 1
number_results = jsonresults['totalItems']
except HTTPError as err:
logger.warn(
'Google Books API Error [%s]: Check your API key or wait a while'
% err.reason)
break
if number_results == 0:
logger.warn('Found no results for %s' % authorname)
break
else:
logger.debug('Found %s result%s for %s' %
(number_results, plural(number_results),
authorname))
startindex += 40
for item in jsonresults['items']:
total_count += 1
# skip if no author, no author is no book.
try:
_ = item['volumeInfo']['authors'][0]
except KeyError:
logger.debug(
'Skipped a result without authorfield.')
continue
try:
if item['volumeInfo']['industryIdentifiers'][0][
'type'] == 'ISBN_10':
bookisbn = item['volumeInfo'][
'industryIdentifiers'][0]['identifier']
else:
bookisbn = ""
except KeyError:
bookisbn = ""
isbnhead = ""
if len(bookisbn) == 10:
isbnhead = bookisbn[0:3]
elif len(bookisbn) == 13:
isbnhead = bookisbn[3:6]
try:
booklang = item['volumeInfo']['language']
except KeyError:
booklang = "Unknown"
# do we care about language?
if "All" not in valid_langs:
if bookisbn != "":
# seems google lies to us, sometimes tells us books
# are in english when they are not
if booklang == "Unknown" or booklang == "en":
googlelang = booklang
match = False
lang = myDB.match(
'SELECT lang FROM languages where isbn = "%s"'
% isbnhead)
if lang:
booklang = lang['lang']
cache_hits += 1
logger.debug(
"Found cached language [%s] for [%s]"
% (booklang, isbnhead))
match = True
if not match:
# no match in cache, try lookup dict
if isbnhead:
if len(
bookisbn
) == 13 and bookisbn.startswith(
'979'):
for lang in lazylibrarian.isbn_979_dict:
if isbnhead.startswith(
lang):
booklang = lazylibrarian.isbn_979_dict[
lang]
logger.debug(
"ISBN979 returned %s for %s"
% (booklang,
isbnhead))
match = True
break
elif (len(bookisbn) == 10) or \
(len(bookisbn) == 13 and bookisbn.startswith('978')):
for lang in lazylibrarian.isbn_978_dict:
if isbnhead.startswith(
lang):
booklang = lazylibrarian.isbn_978_dict[
lang]
logger.debug(
"ISBN979 returned %s for %s"
% (booklang,
isbnhead))
match = True
break
if match:
myDB.action(
'insert into languages values ("%s", "%s")'
% (isbnhead, booklang))
logger.debug(u"GB language: " +
booklang)
if not match:
# try searching librarything for a language code using the isbn
# if no language found, librarything return value is "invalid" or "unknown"
# librarything returns plain text, not xml
BOOK_URL = 'http://www.librarything.com/api/thingLang.php?isbn=' + bookisbn
try:
librarything_wait()
resp = urllib2.urlopen(
BOOK_URL, timeout=30).read()
lt_lang_hits += 1
logger.debug(
"LibraryThing reports language [%s] for %s"
% (resp, isbnhead))
if resp != 'invalid' and resp != 'unknown':
booklang = resp # found a language code
match = True
myDB.action(
'insert into languages values ("%s", "%s")'
% (isbnhead, booklang))
logger.debug(u"LT language: " +
booklang)
except Exception as e:
booklang = ""
logger.error(
"Error finding language: %s" %
str(e))
if match:
# We found a better language match
if googlelang == "en" and booklang not in [
"en-US", "en-GB", "eng"
]:
# these are all english, may need to expand this list
booknamealt = item['volumeInfo'][
'title']
logger.debug(
"%s Google thinks [%s], we think [%s]"
% (booknamealt, googlelang,
booklang))
gb_lang_change += 1
else: # No match anywhere, accept google language
booklang = googlelang
# skip if language is in ignore list
if booklang not in valid_langs:
booknamealt = item['volumeInfo']['title']
logger.debug('Skipped [%s] with language %s' %
(booknamealt, booklang))
ignored += 1
continue
try:
bookpub = item['volumeInfo']['publisher']
except KeyError:
bookpub = ""
try:
booksub = item['volumeInfo']['subtitle']
except KeyError:
booksub = ""
if not booksub:
series = ""
seriesNum = ""
else:
try:
series = booksub.split('(')[1].split(
' Series ')[0]
except IndexError:
series = ""
if series.endswith(')'):
series = series[:-1]
try:
seriesNum = booksub.split('(')[1].split(
' Series ')[1].split(')')[0]
if seriesNum[0] == '#':
seriesNum = seriesNum[1:]
except IndexError:
seriesNum = ""
if not seriesNum and '#' in series:
words = series.rsplit('#', 1)
series = words[0].strip()
seriesNum = words[1].strip()
if not seriesNum and ' ' in series:
words = series.rsplit(' ', 1)
# has to be unicode for isnumeric()
if (u"%s" % words[1]).isnumeric():
series = words[0]
seriesNum = words[1]
try:
bookdate = item['volumeInfo']['publishedDate']
except KeyError:
bookdate = '0000-00-00'
try:
bookimg = item['volumeInfo']['imageLinks'][
'thumbnail']
except KeyError:
bookimg = 'images/nocover.png'
try:
bookrate = item['volumeInfo']['averageRating']
except KeyError:
bookrate = 0
try:
bookpages = item['volumeInfo']['pageCount']
except KeyError:
bookpages = 0
try:
bookgenre = item['volumeInfo']['categories'][0]
except KeyError:
bookgenre = ""
try:
bookdesc = item['volumeInfo']['description']
except KeyError:
bookdesc = ""
bookname = item['volumeInfo']['title']
bookname = unaccented(bookname)
dic = {':': '.', '"': '', '\'': ''}
bookname = replace_all(bookname, dic)
bookname = bookname.strip() # strip whitespace
booklink = item['volumeInfo']['canonicalVolumeLink']
bookrate = float(bookrate)
bookid = item['id']
# GoodReads sometimes has multiple bookids for the same book (same author/title, different editions)
# and sometimes uses the same bookid if the book is the same but the title is slightly different
#
# Not sure if googlebooks does too, but we only want one...
existing_book = myDB.match(
'SELECT Status,Manual FROM books WHERE BookID = "%s"'
% bookid)
if existing_book:
book_status = existing_book['Status']
locked = existing_book['Manual']
if locked is None:
locked = False
elif locked.isdigit():
locked = bool(int(locked))
else:
book_status = bookstatus # new_book status, or new_author status
locked = False
rejected = False
check_status = False
if re.match(
'[^\w-]', bookname
): # remove books with bad characters in title
logger.debug(
"[%s] removed book for bad characters" %
bookname)
removedResults += 1
rejected = True
if not rejected and not bookname:
logger.debug(
'Rejecting bookid %s for %s, no bookname' %
(bookid, authorname))
removedResults += 1
rejected = True
if not rejected and lazylibrarian.CONFIG['NO_FUTURE']:
# googlebooks sometimes gives yyyy, sometimes yyyy-mm, sometimes yyyy-mm-dd
if bookdate > today()[:len(bookdate)]:
logger.debug(
'Rejecting %s, future publication date %s'
% (bookname, bookdate))
removedResults += 1
rejected = True
if not rejected:
cmd = 'SELECT BookID FROM books,authors WHERE books.AuthorID = authors.AuthorID'
cmd += ' and BookName = "%s" COLLATE NOCASE and AuthorName = "%s" COLLATE NOCASE'% \
(bookname.replace('"', '""'), authorname.replace('"', '""'))
match = myDB.match(cmd)
if match:
if match['BookID'] != bookid:
# we have a different book with this author/title already
logger.debug(
'Rejecting bookid %s for [%s][%s] already got %s'
% (match['BookID'], authorname,
bookname, bookid))
rejected = True
duplicates += 1
if not rejected:
cmd = 'SELECT AuthorName,BookName FROM books,authors'
cmd += ' WHERE authors.AuthorID = books.AuthorID AND BookID="%s"' % bookid
match = myDB.match(cmd)
if match:
# we have a book with this bookid already
if bookname != match[
'BookName'] or authorname != match[
'AuthorName']:
logger.debug(
'Rejecting bookid %s for [%s][%s] already got bookid for [%s][%s]'
% (bookid, authorname, bookname,
match['AuthorName'],
match['BookName']))
else:
logger.debug(
'Rejecting bookid %s for [%s][%s] already got this book in database'
% (bookid, authorname, bookname))
check_status = True
duplicates += 1
rejected = True
if check_status or not rejected:
if book_status != "Ignored" and not locked:
controlValueDict = {"BookID": bookid}
newValueDict = {
"AuthorID": authorid,
"BookName": bookname,
"BookSub": booksub,
"BookDesc": bookdesc,
"BookIsbn": bookisbn,
"BookPub": bookpub,
"BookGenre": bookgenre,
"BookImg": bookimg,
"BookLink": booklink,
"BookRate": bookrate,
"BookPages": bookpages,
"BookDate": bookdate,
"BookLang": booklang,
"Status": book_status,
"BookAdded": today()
}
resultcount += 1
myDB.upsert("books", newValueDict,
controlValueDict)
logger.debug(u"Book found: " + bookname + " " +
bookdate)
updated = False
if 'nocover' in bookimg or 'nophoto' in bookimg:
# try to get a cover from librarything
workcover = getBookCover(bookid)
if workcover:
logger.debug(
u'Updated cover for %s to %s' %
(bookname, workcover))
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": workcover}
myDB.upsert("books", newValueDict,
controlValueDict)
updated = True
elif bookimg and bookimg.startswith('http'):
link, success = cache_img("book",
bookid,
bookimg,
refresh=refresh)
if success:
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": link}
myDB.upsert("books", newValueDict,
controlValueDict)
updated = True
else:
logger.debug(
'Failed to cache image for %s' %
bookimg)
seriesdict = {}
if lazylibrarian.CONFIG['ADD_SERIES']:
# prefer series info from librarything
seriesdict = getWorkSeries(bookid)
if seriesdict:
logger.debug(
u'Updated series: %s [%s]' %
(bookid, seriesdict))
updated = True
else:
# librarything doesn't have series info. Any in the title?
if series:
seriesdict = {
cleanName(unaccented(series)):
seriesNum
}
setSeries(seriesdict, bookid)
new_status = setStatus(bookid, seriesdict,
bookstatus)
if not new_status == book_status:
book_status = new_status
updated = True
worklink = getWorkPage(bookid)
if worklink:
controlValueDict = {"BookID": bookid}
newValueDict = {"WorkPage": worklink}
myDB.upsert("books", newValueDict,
controlValueDict)
if not existing_book:
logger.debug(
"[%s] Added book: %s [%s] status %s" %
(authorname, bookname, booklang,
book_status))
added_count += 1
elif updated:
logger.debug(
"[%s] Updated book: %s [%s] status %s"
% (authorname, bookname, booklang,
book_status))
updated_count += 1
else:
book_ignore_count += 1
except KeyError:
pass
deleteEmptySeries()
logger.debug(
'[%s] The Google Books API was hit %s time%s to populate book list'
% (authorname, api_hits, plural(api_hits)))
lastbook = myDB.match(
'SELECT BookName, BookLink, BookDate, BookImg from books WHERE AuthorID="%s" \
AND Status != "Ignored" order by BookDate DESC'
% authorid)
if lastbook: # maybe there are no books [remaining] for this author
lastbookname = lastbook['BookName']
lastbooklink = lastbook['BookLink']
lastbookdate = lastbook['BookDate']
lastbookimg = lastbook['BookImg']
else:
lastbookname = ""
lastbooklink = ""
lastbookdate = ""
lastbookimg = ""
controlValueDict = {"AuthorID": authorid}
newValueDict = {
"Status": "Active",
"LastBook": lastbookname,
"LastLink": lastbooklink,
"LastDate": lastbookdate,
"LastBookImg": lastbookimg
}
myDB.upsert("authors", newValueDict, controlValueDict)
logger.debug("Found %s total book%s for author" %
(total_count, plural(total_count)))
logger.debug("Removed %s unwanted language result%s for author" %
(ignored, plural(ignored)))
logger.debug(
"Removed %s bad character or no-name result%s for author" %
(removedResults, plural(removedResults)))
logger.debug("Removed %s duplicate result%s for author" %
(duplicates, plural(duplicates)))
logger.debug("Found %s book%s by author marked as Ignored" %
(book_ignore_count, plural(book_ignore_count)))
logger.debug("Imported/Updated %s book%s for author" %
(resultcount, plural(resultcount)))
myDB.action(
'insert into stats values ("%s", %i, %i, %i, %i, %i, %i, %i, %i, %i)'
% (authorname.replace('"', '""'), api_hits, gr_lang_hits,
lt_lang_hits, gb_lang_change, cache_hits, ignored,
removedResults, not_cached, duplicates))
if refresh:
logger.info(
"[%s] Book processing complete: Added %s book%s / Updated %s book%s"
% (authorname, added_count, plural(added_count),
updated_count, plural(updated_count)))
else:
logger.info(
"[%s] Book processing complete: Added %s book%s to the database"
% (authorname, added_count, plural(added_count)))
except Exception:
logger.error('Unhandled exception in GB.get_author_books: %s' %
traceback.format_exc())
def getBookCover(bookID=None):
""" Return link to a local file containing a book cover image for a bookid.
Try 1. Local file cached from goodreads/googlebooks when book was imported
2. cover.jpg if we have the book
3. LibraryThing whatwork
4. Goodreads search if book was imported from goodreads
5. Google images search
Return None if no cover available. """
if not bookID:
logger.error("getBookCover- No bookID")
return None
cachedir = lazylibrarian.CACHEDIR
coverfile = os.path.join(cachedir, "book", bookID + '.jpg')
if os.path.isfile(coverfile): # use cached image if there is one
lazylibrarian.CACHE_HIT = int(lazylibrarian.CACHE_HIT) + 1
logger.debug(u"getBookCover: Returning Cached response for %s" %
coverfile)
coverlink = 'cache/book/' + bookID + '.jpg'
return coverlink
lazylibrarian.CACHE_MISS = int(lazylibrarian.CACHE_MISS) + 1
myDB = database.DBConnection()
item = myDB.match('select BookFile from books where bookID=?', (bookID, ))
if item:
bookfile = item['BookFile']
if bookfile: # we may have a cover.jpg in the same folder
bookdir = os.path.dirname(bookfile)
coverimg = os.path.join(bookdir, "cover.jpg")
if os.path.isfile(coverimg):
logger.debug(u"getBookCover: Copying book cover to %s" %
coverfile)
shutil.copyfile(coverimg, coverfile)
coverlink = 'cache/book/' + bookID + '.jpg'
return coverlink
# no cover.jpg, try to get a cover from goodreads
cmd = 'select BookName,AuthorName,BookLink from books,authors where bookID=?'
cmd += ' and books.AuthorID = authors.AuthorID'
item = myDB.match(cmd, (bookID, ))
safeparams = ''
if item:
title = safe_unicode(item['BookName'])
title = title.encode(lazylibrarian.SYS_ENCODING)
author = safe_unicode(item['AuthorName'])
author = author.encode(lazylibrarian.SYS_ENCODING)
booklink = item['BookLink']
safeparams = urllib.quote_plus("%s %s" % (author, title))
if 'goodreads' in booklink:
# if the bookID is a goodreads one, we can call https://www.goodreads.com/book/show/{bookID}
# and scrape the page for og:image
# <meta property="og:image" content="https://i.gr-assets.com/images/S/photo.goodreads.com/books/
# 1388267702i/16304._UY475_SS475_.jpg"/>
# to get the cover
time_now = int(time.time())
if time_now <= lazylibrarian.LAST_GOODREADS:
time.sleep(1)
lazylibrarian.LAST_GOODREADS = time_now
result, success = fetchURL(booklink)
if success:
try:
img = result.split('id="coverImage"')[1].split(
'src="')[1].split('"')[0]
except IndexError:
try:
img = result.split('og:image')[1].split('="')[1].split(
'"')[0]
except IndexError:
img = None
if img and img.startswith(
'http'
) and 'nocover' not in img and 'nophoto' not in img:
time_now = int(time.time())
if time_now <= lazylibrarian.LAST_GOODREADS:
time.sleep(1)
lazylibrarian.LAST_GOODREADS = time_now
coverlink, success = cache_img("book", bookID, img)
if success:
logger.debug(
"getBookCover: Caching goodreads cover for %s %s" %
(item['AuthorName'], item['BookName']))
return coverlink
else:
logger.debug(
"getBookCover: Error getting goodreads image for %s, [%s]"
% (img, coverlink))
else:
logger.debug(
"getBookCover: No image found in goodreads page for %s"
% bookID)
else:
logger.debug("getBookCover: Error getting page %s, [%s]" %
(booklink, result))
# nothing from goodreads, see if librarything workpage has a cover
work = getBookWork(bookID, "Cover")
if work:
try:
img = work.split('workCoverImage')[1].split('="')[1].split('"')[0]
if img and img.startswith('http'):
coverlink, success = cache_img("book", bookID, img)
if success:
logger.debug(
u"getBookCover: Caching librarything cover for %s" %
bookID)
return coverlink
else:
logger.debug(
'getBookCover: Failed to cache image for %s [%s]' %
(img, coverlink))
else:
logger.debug(
"getBookCover: No image found in work page for %s" %
bookID)
except IndexError:
logger.debug('getBookCover: Image not found in work page for %s' %
bookID)
try:
img = work.split('og:image')[1].split('="')[1].split('"')[0]
if img and img.startswith('http'):
coverlink, success = cache_img("book", bookID, img)
if success:
logger.debug(
u"getBookCover: Caching librarything cover for %s" %
bookID)
return coverlink
else:
logger.debug(
'getBookCover: Failed to cache image for %s [%s]' %
(img, coverlink))
else:
logger.debug(
"getBookCover: No image found in work page for %s" %
bookID)
except IndexError:
logger.debug('getBookCover: Image not found in work page for %s' %
bookID)
if safeparams:
# if all else fails, try a google image search...
# tbm=isch search images
# tbs=isz:l large images
# ift:jpg jpeg file type
URL = "https://www.google.com/search?tbm=isch&tbs=isz:l,ift:jpg&as_q=" + safeparams + "+ebook"
result, success = fetchURL(URL)
if success:
try:
img = result.split('url?q=')[1].split('">')[1].split(
'src="')[1].split('"')[0]
except IndexError:
img = None
if img and img.startswith('http'):
coverlink, success = cache_img("book", bookID, img)
if success:
logger.debug(
"getBookCover: Caching google cover for %s %s" %
(item['AuthorName'], item['BookName']))
return coverlink
else:
logger.debug(
"getBookCover: Error getting google image %s, [%s]" %
(img, coverlink))
else:
logger.debug(
"getBookCover: No image found in google page for %s" %
bookID)
else:
logger.debug(
"getBookCover: Error getting google page for %s, [%s]" %
(safeparams, result))
return None
def find_book(self, bookid=None, bookstatus="None"):
myDB = database.DBConnection()
if not lazylibrarian.CONFIG['GB_API']:
logger.warn('No GoogleBooks API key, check config')
URL = 'https://www.googleapis.com/books/v1/volumes/' + \
str(bookid) + "?key=" + lazylibrarian.CONFIG['GB_API']
jsonresults, in_cache = gb_json_request(URL)
if jsonresults is None:
logger.debug('No results found for %s' % bookid)
return
if not bookstatus:
bookstatus = lazylibrarian.CONFIG['NEWBOOK_STATUS']
book = bookdict(jsonresults)
dic = {':': '.', '"': '', '\'': ''}
bookname = replace_all(book['name'], dic)
bookname = unaccented(bookname)
bookname = bookname.strip() # strip whitespace
if not book['author']:
logger.debug('Book %s does not contain author field, skipping' %
bookname)
return
# warn if language is in ignore list, but user said they wanted this book
valid_langs = getList(lazylibrarian.CONFIG['IMP_PREFLANG'])
if book['lang'] not in valid_langs and 'All' not in valid_langs:
logger.debug(
'Book %s googlebooks language does not match preference, %s' %
(bookname, book['lang']))
if lazylibrarian.CONFIG['NO_PUBDATE']:
if not book['date'] or book['date'] == '0000':
logger.warn(
'Book %s Publication date does not match preference, %s' %
(bookname, book['date']))
if lazylibrarian.CONFIG['NO_FUTURE']:
if book['date'] > today()[:4]:
logger.warn(
'Book %s Future publication date does not match preference, %s'
% (bookname, book['date']))
authorname = book['author']
GR = GoodReads(authorname)
author = GR.find_author_id()
if author:
AuthorID = author['authorid']
match = myDB.match('SELECT AuthorID from authors WHERE AuthorID=?',
(AuthorID, ))
if not match:
match = myDB.match(
'SELECT AuthorID from authors WHERE AuthorName=?',
(author['authorname'], ))
if match:
logger.debug(
'%s: Changing authorid from %s to %s' %
(author['authorname'], AuthorID, match['AuthorID']))
AuthorID = match[
'AuthorID'] # we have a different authorid for that authorname
else: # no author but request to add book, add author with newauthor status
# User hit "add book" button from a search or a wishlist import
newauthor_status = 'Active'
if lazylibrarian.CONFIG['NEWAUTHOR_STATUS'] in [
'Skipped', 'Ignored'
]:
newauthor_status = 'Paused'
controlValueDict = {"AuthorID": AuthorID}
newValueDict = {
"AuthorName": author['authorname'],
"AuthorImg": author['authorimg'],
"AuthorLink": author['authorlink'],
"AuthorBorn": author['authorborn'],
"AuthorDeath": author['authordeath'],
"DateAdded": today(),
"Status": newauthor_status
}
authorname = author['authorname']
myDB.upsert("authors", newValueDict, controlValueDict)
if lazylibrarian.CONFIG['NEWAUTHOR_BOOKS']:
self.get_author_books(AuthorID,
entrystatus=lazylibrarian.
CONFIG['NEWAUTHOR_STATUS'])
else:
logger.warn("No AuthorID for %s, unable to add book %s" %
(book['author'], bookname))
return
controlValueDict = {"BookID": bookid}
newValueDict = {
"AuthorID": AuthorID,
"BookName": bookname,
"BookSub": book['sub'],
"BookDesc": book['desc'],
"BookIsbn": book['isbn'],
"BookPub": book['pub'],
"BookGenre": book['genre'],
"BookImg": book['img'],
"BookLink": book['link'],
"BookRate": float(book['rate']),
"BookPages": book['pages'],
"BookDate": book['date'],
"BookLang": book['lang'],
"Status": bookstatus,
"AudioStatus": lazylibrarian.CONFIG['NEWAUDIO_STATUS'],
"BookAdded": today()
}
myDB.upsert("books", newValueDict, controlValueDict)
logger.info("%s by %s added to the books database" %
(bookname, authorname))
if 'nocover' in book['img'] or 'nophoto' in book['img']:
# try to get a cover from another source
workcover, source = getBookCover(bookid)
if workcover:
logger.debug('Updated cover for %s using %s' %
(bookname, source))
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": workcover}
myDB.upsert("books", newValueDict, controlValueDict)
elif book['img'] and book['img'].startswith('http'):
link, success, _ = cache_img("book", bookid, book['img'])
if success:
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": link}
myDB.upsert("books", newValueDict, controlValueDict)
else:
logger.debug('Failed to cache image for %s' % book['img'])
serieslist = []
if book['series']:
serieslist = [('', book['seriesNum'],
cleanName(unaccented(book['series']), '&/'))]
if lazylibrarian.CONFIG['ADD_SERIES']:
newserieslist = getWorkSeries(bookid)
if newserieslist:
serieslist = newserieslist
logger.debug('Updated series: %s [%s]' % (bookid, serieslist))
setSeries(serieslist, bookid)
worklink = getWorkPage(bookid)
if worklink:
controlValueDict = {"BookID": bookid}
newValueDict = {"WorkPage": worklink}
myDB.upsert("books", newValueDict, controlValueDict)
def LibraryScan(startdir=None):
""" Scan a directory tree adding new books into database
Return how many books you added """
try:
destdir = lazylibrarian.DIRECTORY('Destination')
if not startdir:
if not destdir:
logger.warn('Cannot find destination directory: %s. Not scanning' % destdir)
return 0
startdir = destdir
if not os.path.isdir(startdir):
logger.warn('Cannot find directory: %s. Not scanning' % startdir)
return 0
if not internet():
logger.warn('Libraryscan: No internet connection')
return 0
myDB = database.DBConnection()
# keep statistics of full library scans
if startdir == destdir:
myDB.action('DELETE from stats')
try: # remove any extra whitespace in authornames
authors = myDB.select('SELECT AuthorID,AuthorName FROM authors WHERE AuthorName like "% %"')
if authors:
logger.info('Removing extra spaces from %s authorname%s' % (len(authors), plural(len(authors))))
for author in authors:
authorid = author["AuthorID"]
authorname = ' '.join(author['AuthorName'].split())
# Have we got author name both with-and-without extra spaces? If so, merge them
duplicate = myDB.match(
'Select AuthorID,AuthorName FROM authors WHERE AuthorName="%s"' % authorname)
if duplicate:
myDB.action('DELETE from authors where authorname="%s"' % author['AuthorName'])
if author['AuthorID'] != duplicate['AuthorID']:
myDB.action('UPDATE books set AuthorID="%s" WHERE AuthorID="%s"' %
(duplicate['AuthorID'], author['AuthorID']))
else:
myDB.action(
'UPDATE authors set AuthorName="%s" WHERE AuthorID="%s"' % (authorname, authorid))
except Exception as e:
logger.info('Error: ' + str(e))
logger.info('Scanning ebook directory: %s' % startdir)
new_book_count = 0
modified_count = 0
rescan_count = 0
rescan_hits = 0
file_count = 0
author = ""
if lazylibrarian.CONFIG['FULL_SCAN']:
cmd = 'select AuthorName, BookName, BookFile, BookID from books,authors'
cmd += ' where books.AuthorID = authors.AuthorID and books.Status="Open"'
if not startdir == destdir:
cmd += ' and BookFile like "' + startdir + '%"'
books = myDB.select(cmd)
status = lazylibrarian.CONFIG['NOTFOUND_STATUS']
logger.info('Missing books will be marked as %s' % status)
for book in books:
bookID = book['BookID']
bookfile = book['BookFile']
if not (bookfile and os.path.isfile(bookfile)):
myDB.action('update books set Status="%s" where BookID="%s"' % (status, bookID))
myDB.action('update books set BookFile="" where BookID="%s"' % bookID)
logger.warn('Book %s - %s updated as not found on disk' % (book['AuthorName'], book['BookName']))
# to save repeat-scans of the same directory if it contains multiple formats of the same book,
# keep track of which directories we've already looked at
processed_subdirectories = []
warned = False # have we warned about no new authors setting
matchString = ''
for char in lazylibrarian.CONFIG['EBOOK_DEST_FILE']:
matchString = matchString + '\\' + char
# massage the EBOOK_DEST_FILE config parameter into something we can use
# with regular expression matching
booktypes = ''
count = -1
booktype_list = getList(lazylibrarian.CONFIG['EBOOK_TYPE'])
for book_type in booktype_list:
count += 1
if count == 0:
booktypes = book_type
else:
booktypes = booktypes + '|' + book_type
matchString = matchString.replace("\\$\\A\\u\\t\\h\\o\\r", "(?P<author>.*?)").replace(
"\\$\\T\\i\\t\\l\\e", "(?P<book>.*?)") + '\.[' + booktypes + ']'
pattern = re.compile(matchString, re.VERBOSE)
for r, d, f in os.walk(startdir):
for directory in d[:]:
# prevent magazine being scanned
if directory.startswith("_") or directory.startswith("."):
d.remove(directory)
for files in f:
file_count += 1
if isinstance(r, str):
r = r.decode(lazylibrarian.SYS_ENCODING)
subdirectory = r.replace(startdir, '')
# Added new code to skip if we've done this directory before.
# Made this conditional with a switch in config.ini
# in case user keeps multiple different books in the same subdirectory
if lazylibrarian.CONFIG['IMP_SINGLEBOOK'] and (subdirectory in processed_subdirectories):
logger.debug("[%s] already scanned" % subdirectory)
else:
# If this is a book, try to get author/title/isbn/language
# if epub or mobi, read metadata from the book
# If metadata.opf exists, use that allowing it to override
# embedded metadata. User may have edited metadata.opf
# to merge author aliases together
# If all else fails, try pattern match for author/title
# and look up isbn/lang from LT or GR later
match = 0
if is_valid_booktype(files):
logger.debug("[%s] Now scanning subdirectory %s" % (startdir, subdirectory))
language = "Unknown"
isbn = ""
book = ""
author = ""
gr_id = ""
gb_id = ""
extn = os.path.splitext(files)[1]
# if it's an epub or a mobi we can try to read metadata from it
if (extn == ".epub") or (extn == ".mobi"):
book_filename = os.path.join(r, files).encode(lazylibrarian.SYS_ENCODING)
try:
res = get_book_info(book_filename)
except Exception as e:
logger.debug('get_book_info failed for %s, %s' % (book_filename, str(e)))
res = {}
# title and creator are the minimum we need
if 'title' in res and 'creator' in res:
book = res['title']
author = res['creator']
if book and len(book) > 2 and author and len(author) > 2:
match = 1
if 'language' in res:
language = res['language']
if 'identifier' in res:
isbn = res['identifier']
if 'type' in res:
extn = res['type']
logger.debug("book meta [%s] [%s] [%s] [%s] [%s]" %
(isbn, language, author, book, extn))
if not match:
logger.debug("Book meta incomplete in %s" % book_filename)
# calibre uses "metadata.opf", LL uses "bookname - authorname.opf"
# just look for any .opf file in the current directory since we don't know
# LL preferred authorname/bookname at this point.
# Allow metadata in file to override book contents as may be users pref
metafile = opf_file(r)
try:
res = get_book_info(metafile)
except Exception as e:
logger.debug('get_book_info failed for %s, %s' % (metafile, str(e)))
res = {}
# title and creator are the minimum we need
if 'title' in res and 'creator' in res:
book = res['title']
author = res['creator']
if book and len(book) > 2 and author and len(author) > 2:
match = 1
if 'language' in res:
language = res['language']
if 'identifier' in res:
isbn = res['identifier']
if 'gr_id' in res:
gr_id = res['gr_id']
logger.debug("file meta [%s] [%s] [%s] [%s] [%s]" % (isbn, language, author, book, gr_id))
if not match:
logger.debug("File meta incomplete in %s" % metafile)
if not match: # no author/book from metadata file, and not embedded either
match = pattern.match(files)
if match:
author = match.group("author")
book = match.group("book")
if len(book) <= 2 or len(author) <= 2:
match = 0
if not match:
logger.debug("Pattern match failed [%s]" % files)
if match:
# flag that we found a book in this subdirectory
processed_subdirectories.append(subdirectory)
# If we have a valid looking isbn, and language != "Unknown", add it to cache
if language != "Unknown" and is_valid_isbn(isbn):
logger.debug("Found Language [%s] ISBN [%s]" % (language, isbn))
# we need to add it to language cache if not already
# there, is_valid_isbn has checked length is 10 or 13
if len(isbn) == 10:
isbnhead = isbn[0:3]
else:
isbnhead = isbn[3:6]
match = myDB.match('SELECT lang FROM languages where isbn = "%s"' % isbnhead)
if not match:
myDB.action('insert into languages values ("%s", "%s")' % (isbnhead, language))
logger.debug("Cached Lang [%s] ISBN [%s]" % (language, isbnhead))
else:
logger.debug("Already cached Lang [%s] ISBN [%s]" % (language, isbnhead))
author, authorid, new = addAuthorNameToDB(author) # get the author name as we know it...
if author:
# author exists, check if this book by this author is in our database
# metadata might have quotes in book name
# some books might be stored under a different author name
# eg books by multiple authors, books where author is "writing as"
# or books we moved to "merge" authors
book = book.replace("'", "")
# First try and find it under author and bookname
# as we may have it under a different bookid or isbn to goodreads/googlebooks
# which might have several bookid/isbn for the same book
bookid = find_book_in_db(myDB, author, book)
if not bookid:
# Title or author name might not match or multiple authors
# See if the gr_id, gb_id is already in our database
if gr_id:
bookid = gr_id
elif gb_id:
bookid = gb_id
else:
bookid = ""
if bookid:
match = myDB.match('SELECT BookID FROM books where BookID = "%s"' % bookid)
if not match:
msg = 'Unable to find book %s by %s in database, trying to add it using '
if bookid == gr_id:
msg += "GoodReads ID " + gr_id
if bookid == gb_id:
msg += "GoogleBooks ID " + gb_id
logger.debug(msg % (book, author))
if lazylibrarian.CONFIG['BOOK_API'] == "GoodReads" and gr_id:
GR_ID = GoodReads(gr_id)
GR_ID.find_book(gr_id, None)
elif lazylibrarian.CONFIG['BOOK_API'] == "GoogleBooks" and gb_id:
GB_ID = GoogleBooks(gb_id)
GB_ID.find_book(gb_id, None)
# see if it's there now...
match = myDB.match('SELECT BookID from books where BookID="%s"' % bookid)
if not match:
logger.debug("Unable to add bookid %s to database" % bookid)
bookid = ""
if not bookid and isbn:
# See if the isbn is in our database
match = myDB.match('SELECT BookID FROM books where BookIsbn = "%s"' % isbn)
if match:
bookid = match['BookID']
if not bookid:
# get author name from parent directory of this book directory
newauthor = os.path.basename(os.path.dirname(r))
# calibre replaces trailing periods with _ eg Smith Jr. -> Smith Jr_
if newauthor.endswith('_'):
newauthor = newauthor[:-1] + '.'
if author.lower() != newauthor.lower():
logger.debug("Trying authorname [%s]" % newauthor)
bookid = find_book_in_db(myDB, newauthor, book)
if bookid:
logger.warn("%s not found under [%s], found under [%s]" %
(book, author, newauthor))
# at this point if we still have no bookid, it looks like we
# have author and book title but no database entry for it
if not bookid:
if lazylibrarian.CONFIG['BOOK_API'] == "GoodReads":
# Either goodreads doesn't have the book or it didn't match language prefs
# Since we have the book anyway, try and reload it ignoring language prefs
rescan_count += 1
base_url = 'http://www.goodreads.com/search.xml?q='
params = {"key": lazylibrarian.CONFIG['GR_API']}
if author[1] in '. ':
surname = author
forename = ''
while surname[1] in '. ':
forename = forename + surname[0] + '.'
surname = surname[2:].strip()
if author != forename + ' ' + surname:
logger.debug('Stripped authorname [%s] to [%s %s]' %
(author, forename, surname))
author = forename + ' ' + surname
author = ' '.join(author.split()) # ensure no extra whitespace
searchname = author + ' ' + book
searchname = cleanName(unaccented(searchname))
searchterm = urllib.quote_plus(searchname.encode(lazylibrarian.SYS_ENCODING))
set_url = base_url + searchterm + '&' + urllib.urlencode(params)
try:
rootxml, in_cache = get_xml_request(set_url)
if not len(rootxml):
logger.debug("Error requesting results from GoodReads")
else:
resultxml = rootxml.getiterator('work')
for item in resultxml:
booktitle = item.find('./best_book/title').text
book_fuzz = fuzz.token_set_ratio(booktitle, book)
if book_fuzz >= 98:
logger.debug("Rescan found %s : %s" % (booktitle, language))
rescan_hits += 1
bookid = item.find('./best_book/id').text
GR_ID = GoodReads(bookid)
GR_ID.find_book(bookid, None)
if language and language != "Unknown":
# set language from book metadata
logger.debug("Setting language from metadata %s : %s" % (booktitle, language))
myDB.action('UPDATE books SET BookLang="%s" WHERE BookID="%s"' %
(language, bookid))
break
if not bookid:
logger.warn("GoodReads doesn't know about %s" % book)
except Exception as e:
logger.error("Error finding rescan results: %s" % str(e))
elif lazylibrarian.CONFIG['BOOK_API'] == "GoogleBooks":
# if we get here using googlebooks it's because googlebooks
# doesn't have the book. No point in looking for it again.
logger.warn("GoogleBooks doesn't know about %s" % book)
# see if it's there now...
if bookid:
cmd = 'SELECT books.Status, BookFile, AuthorName, BookName from books,authors '
cmd += 'where books.AuthorID = authors.AuthorID and BookID="%s"' % bookid
check_status = myDB.match(cmd)
if not check_status:
logger.debug('Unable to find bookid %s in database' % bookid)
else:
if check_status['Status'] != 'Open':
# we found a new book
new_book_count += 1
myDB.action(
'UPDATE books set Status="Open" where BookID="%s"' % bookid)
# store book location so we can check if it gets removed
book_filename = os.path.join(r, files)
if not check_status['BookFile']: # no previous location
myDB.action('UPDATE books set BookFile="%s" where BookID="%s"' %
(book_filename, bookid))
# location may have changed since last scan
elif book_filename != check_status['BookFile']:
modified_count += 1
logger.warn("Updating book location for %s %s from %s to %s" %
(author, book, check_status['BookFile'], book_filename))
logger.debug("%s %s matched %s BookID %s, [%s][%s]" %
(author, book, check_status['Status'], bookid,
check_status['AuthorName'], check_status['BookName']))
myDB.action('UPDATE books set BookFile="%s" where BookID="%s"' %
(book_filename, bookid))
# update cover file to cover.jpg in book folder (if exists)
bookdir = os.path.dirname(book_filename)
coverimg = os.path.join(bookdir, 'cover.jpg')
if os.path.isfile(coverimg):
cachedir = lazylibrarian.CACHEDIR
cacheimg = os.path.join(cachedir, 'book', bookid + '.jpg')
copyfile(coverimg, cacheimg)
else:
logger.warn(
"Failed to match book [%s] by [%s] in database" % (book, author))
else:
if not warned and not lazylibrarian.CONFIG['ADD_AUTHOR']:
logger.warn("Add authors to database is disabled")
warned = True
logger.info("%s/%s new/modified book%s found and added to the database" %
(new_book_count, modified_count, plural(new_book_count + modified_count)))
logger.info("%s file%s processed" % (file_count, plural(file_count)))
if startdir == destdir:
# On full library scans, check for missing workpages
setWorkPages()
# and books with unknown language
nolang = myDB.match(
"select count('BookID') as counter from Books where status='Open' and BookLang='Unknown'")
nolang = nolang['counter']
if nolang:
logger.warn("Found %s book%s in your library with unknown language" % (nolang, plural(nolang)))
# show stats if new books were added
stats = myDB.match(
"SELECT sum(GR_book_hits), sum(GR_lang_hits), sum(LT_lang_hits), sum(GB_lang_change), \
sum(cache_hits), sum(bad_lang), sum(bad_char), sum(uncached), sum(duplicates) FROM stats")
st= {'GR_book_hits': stats['sum(GR_book_hits)'], 'GB_book_hits': stats['sum(GR_book_hits)'],
'GR_lang_hits': stats['sum(GR_lang_hits)'], 'LT_lang_hits': stats['sum(LT_lang_hits)'],
'GB_lang_change': stats['sum(GB_lang_change)'], 'cache_hits': stats['sum(cache_hits)'],
'bad_lang': stats['sum(bad_lang)'], 'bad_char': stats['sum(bad_char)'],
'uncached': stats['sum(uncached)'], 'duplicates': stats['sum(duplicates)']}
for item in st.keys():
if st[item] is None:
st[item] = 0
if lazylibrarian.CONFIG['BOOK_API'] == "GoogleBooks":
logger.debug("GoogleBooks was hit %s time%s for books" %
(st['GR_book_hits'], plural(st['GR_book_hits'])))
logger.debug("GoogleBooks language was changed %s time%s" %
(st['GB_lang_change'], plural(st['GB_lang_change'])))
if lazylibrarian.CONFIG['BOOK_API'] == "GoodReads":
logger.debug("GoodReads was hit %s time%s for books" %
(st['GR_book_hits'], plural(st['GR_book_hits'])))
logger.debug("GoodReads was hit %s time%s for languages" %
(st['GR_lang_hits'], plural(st['GR_lang_hits'])))
logger.debug("LibraryThing was hit %s time%s for languages" %
(st['LT_lang_hits'], plural(st['LT_lang_hits'])))
logger.debug("Language cache was hit %s time%s" %
(st['cache_hits'], plural(st['cache_hits'])))
logger.debug("Unwanted language removed %s book%s" %
(st['bad_lang'], plural(st['bad_lang'])))
logger.debug("Unwanted characters removed %s book%s" %
(st['bad_char'], plural(st['bad_char'])))
logger.debug("Unable to cache language for %s book%s with missing ISBN" %
(st['uncached'], plural(st['uncached'])))
logger.debug("Found %s duplicate book%s" %
(st['duplicates'], plural(st['duplicates'])))
logger.debug("Rescan %s hit%s, %s miss" %
(rescan_hits, plural(rescan_hits), rescan_count - rescan_hits))
logger.debug("Cache %s hit%s, %s miss" %
(lazylibrarian.CACHE_HIT, plural(lazylibrarian.CACHE_HIT), lazylibrarian.CACHE_MISS))
cachesize = myDB.match("select count('ISBN') as counter from languages")
logger.debug("ISBN Language cache holds %s entries" % cachesize['counter'])
# Cache any covers and images
images = myDB.select('select bookid, bookimg, bookname from books where bookimg like "http%"')
if len(images):
logger.info("Caching cover%s for %i book%s" % (plural(len(images)), len(images), plural(len(images))))
for item in images:
bookid = item['bookid']
bookimg = item['bookimg']
# bookname = item['bookname']
newimg, success = cache_img("book", bookid, bookimg)
if success:
myDB.action('update books set BookImg="%s" where BookID="%s"' % (newimg, bookid))
images = myDB.select('select AuthorID, AuthorImg, AuthorName from authors where AuthorImg like "http%"')
if len(images):
logger.info("Caching image%s for %i author%s" % (plural(len(images)), len(images), plural(len(images))))
for item in images:
authorid = item['authorid']
authorimg = item['authorimg']
# authorname = item['authorname']
newimg, success = cache_img("author", authorid, authorimg)
if success:
myDB.action('update authors set AuthorImg="%s" where AuthorID="%s"' % (newimg, authorid))
# On full scan, update bookcounts for all authors, not just new ones - refresh may have located
# new books for existing authors especially if switched provider gb/gr or changed wanted languages
authors = myDB.select('select AuthorID from authors')
else:
# On single author/book import, just update bookcount for that author
authors = myDB.select('select AuthorID from authors where AuthorName = "%s"' % author.replace('"', '""'))
logger.debug('Updating bookcounts for %i author%s' % (len(authors), plural(len(authors))))
for author in authors:
update_totals(author['AuthorID'])
logger.info('Library scan complete')
return new_book_count
except Exception:
logger.error('Unhandled exception in libraryScan: %s' % traceback.format_exc())
def addAuthorToDB(authorname=None, refresh=False, authorid=None, addbooks=True):
"""
Add an author to the database by name or id, and optionally get a list of all their books
If author already exists in database, refresh their details and optionally booklist
"""
threadname = threading.currentThread().name
if "Thread-" in threadname:
threading.currentThread().name = "AddAuthorToDB"
# noinspection PyBroadException
try:
myDB = database.DBConnection()
match = False
author = None
authorimg = ''
new_author = not refresh
entry_status = ''
if authorid:
dbauthor = myDB.match("SELECT * from authors WHERE AuthorID=?", (authorid,))
if not dbauthor:
authorname = 'unknown author'
logger.debug("Adding new author id %s to database" % authorid)
new_author = True
else:
entry_status = dbauthor['Status']
authorname = dbauthor['authorname']
logger.debug("Updating author %s " % authorname)
new_author = False
controlValueDict = {"AuthorID": authorid}
newValueDict = {"Status": "Loading"}
if new_author:
newValueDict["AuthorName"] = "Loading"
newValueDict["AuthorImg"] = "images/nophoto.png"
myDB.upsert("authors", newValueDict, controlValueDict)
GR = GoodReads(authorid)
author = GR.get_author_info(authorid=authorid)
if author:
authorname = author['authorname']
authorimg = author['authorimg']
controlValueDict = {"AuthorID": authorid}
newValueDict = {
"AuthorLink": author['authorlink'],
"DateAdded": today()
}
if not dbauthor or (dbauthor and not dbauthor['manual']):
newValueDict["AuthorImg"] = author['authorimg']
newValueDict["AuthorBorn"] = author['authorborn']
newValueDict["AuthorDeath"] = author['authordeath']
if not dbauthor:
newValueDict["AuthorName"] = author['authorname']
elif dbauthor['authorname'] != author['authorname']:
authorname = dbauthor['authorname']
logger.warn("Authorname mismatch for %s [%s][%s]" %
(authorid, dbauthor['authorname'], author['authorname']))
myDB.upsert("authors", newValueDict, controlValueDict)
match = True
else:
logger.warn("Nothing found for %s" % authorid)
if not dbauthor:
myDB.action('DELETE from authors WHERE AuthorID=?', (authorid,))
if authorname and author and not match:
authorname = ' '.join(authorname.split()) # ensure no extra whitespace
GR = GoodReads(authorname)
author = GR.find_author_id(refresh=refresh)
dbauthor = myDB.match("SELECT * from authors WHERE AuthorName=?", (authorname,))
if author and not dbauthor: # may have different name for same authorid (spelling?)
dbauthor = myDB.match("SELECT * from authors WHERE AuthorID=?", (author['authorid'],))
authorname = dbauthor['AuthorName']
controlValueDict = {"AuthorName": authorname}
if not dbauthor:
newValueDict = {
"AuthorID": "0: %s" % authorname,
"Status": "Loading"
}
logger.debug("Now adding new author: %s to database" % authorname)
entry_status = lazylibrarian.CONFIG['NEWAUTHOR_STATUS']
new_author = True
else:
newValueDict = {"Status": "Loading"}
logger.debug("Now updating author: %s" % authorname)
entry_status = dbauthor['Status']
new_author = False
myDB.upsert("authors", newValueDict, controlValueDict)
if author:
authorid = author['authorid']
authorimg = author['authorimg']
controlValueDict = {"AuthorName": authorname}
newValueDict = {
"AuthorID": author['authorid'],
"AuthorLink": author['authorlink'],
"DateAdded": today(),
"Status": "Loading"
}
if dbauthor:
if authorname != dbauthor['authorname']:
# name change might be users preference
logger.warn("Conflicting authorname for %s [%s][%s] Ignoring change" %
(author['authorid'], authorname, dbauthor['authorname']))
authorname = dbauthor['authorname']
# cmd = 'UPDATE authors SET AuthorName=? WHERE AuthorName=?'
# myDB.action(cmd, (author['authorname'], dbauthor['authorname']))
if author['authorid'] != dbauthor['authorid']:
# GoodReads may have altered authorid?
logger.warn("Conflicting authorid for %s (%s:%s) Moving to new authorid" %
(authorname, author['authorid'], dbauthor['authorid']))
cmd = 'UPDATE books SET AuthorID=? WHERE AuthorID=?'
myDB.action(cmd, (author['authorid'], dbauthor['authorid']))
myDB.action('DELETE from authors WHERE AuthorID=?', (dbauthor['authorid'],))
dbauthor = None
if not dbauthor or (dbauthor and not dbauthor['manual']):
newValueDict["AuthorImg"] = author['authorimg']
newValueDict["AuthorBorn"] = author['authorborn']
newValueDict["AuthorDeath"] = author['authordeath']
myDB.upsert("authors", newValueDict, controlValueDict)
match = True
else:
logger.warn("Nothing found for %s" % authorname)
if not dbauthor:
myDB.action('DELETE from authors WHERE AuthorName=?', (authorname,))
return
if not match:
logger.error("No matching result for authorname or authorid")
return
# if author is set to manual, should we allow replacing 'nophoto' ?
new_img = False
match = myDB.match("SELECT Manual from authors WHERE AuthorID=?", (authorid,))
if not match or not match['Manual']:
if authorimg and 'nophoto' in authorimg:
newimg = getAuthorImage(authorid)
if newimg:
authorimg = newimg
new_img = True
# allow caching
if authorimg and authorimg.startswith('http'):
newimg, success, _ = cache_img("author", authorid, authorimg, refresh=refresh)
if success:
authorimg = newimg
new_img = True
else:
logger.debug('Failed to cache image for %s' % authorimg)
if new_img:
controlValueDict = {"AuthorID": authorid}
newValueDict = {"AuthorImg": authorimg}
myDB.upsert("authors", newValueDict, controlValueDict)
if addbooks:
if new_author:
bookstatus = lazylibrarian.CONFIG['NEWAUTHOR_STATUS']
audiostatus = lazylibrarian.CONFIG['NEWAUTHOR_AUDIO']
else:
bookstatus = lazylibrarian.CONFIG['NEWBOOK_STATUS']
audiostatus = lazylibrarian.CONFIG['NEWAUDIO_STATUS']
if entry_status not in ['Active', 'Wanted', 'Ignored', 'Paused']:
entry_status = 'Active' # default for invalid/unknown or "loading"
# process books
if lazylibrarian.CONFIG['BOOK_API'] == "GoogleBooks":
if lazylibrarian.CONFIG['GB_API']:
book_api = GoogleBooks()
book_api.get_author_books(authorid, authorname, bookstatus=bookstatus,
audiostatus=audiostatus, entrystatus=entry_status,
refresh=refresh)
# if lazylibrarian.CONFIG['GR_API']:
# book_api = GoodReads(authorname)
# book_api.get_author_books(authorid, authorname, bookstatus=bookstatus,
# ausiostatus=audiostatus, entrystatus=entry_status,
# refresh=refresh)
elif lazylibrarian.CONFIG['BOOK_API'] == "GoodReads":
if lazylibrarian.CONFIG['GR_API']:
book_api = GoodReads(authorname)
book_api.get_author_books(authorid, authorname, bookstatus=bookstatus,
audiostatus=audiostatus, entrystatus=entry_status,
refresh=refresh)
# if lazylibrarian.CONFIG['GB_API']:
# book_api = GoogleBooks()
# book_api.get_author_books(authorid, authorname, bookstatus=bookstatus,
# audiostatus=audiostatus, entrystatus=entry_status,
# refresh=refresh)
update_totals(authorid)
if new_author and lazylibrarian.CONFIG['GR_FOLLOWNEW']:
res = grfollow(authorid, True)
if res.startswith('Unable'):
logger.warn(res)
try:
followid = res.split("followid=")[1]
logger.debug('%s marked followed' % authorname)
except IndexError:
followid = ''
myDB.action('UPDATE authors SET GRfollow=? WHERE AuthorID=?', (followid, authorid))
else:
# if we're not loading any books, mark author as ignored
entry_status = 'Ignored'
controlValueDict = {"AuthorID": authorid}
newValueDict = {"Status": entry_status}
myDB.upsert("authors", newValueDict, controlValueDict)
msg = "[%s] Author update complete, status %s" % (authorname, entry_status)
logger.info(msg)
return msg
except Exception:
msg = 'Unhandled exception in addAuthorToDB: %s' % traceback.format_exc()
logger.error(msg)
return msg
def getBookCover(bookID=None, src=None):
""" Return link to a local file containing a book cover image for a bookid, and which source used.
Try 1. Local file cached from goodreads/googlebooks when book was imported
2. cover.jpg if we have the book
3. LibraryThing cover image (if you have a dev key)
4. LibraryThing whatwork (if available)
5. Goodreads search (if book was imported from goodreads)
6. Google isbn search (if google has a link to book for sale)
7. Google images search (if lazylibrarian config allows)
src = cache, cover, goodreads, librarything, whatwork, googleisbn, googleimage
Return None if no cover available. """
if not bookID:
logger.error("getBookCover- No bookID")
return None, src
if not src:
src = ''
logger.debug("Getting %s cover for %s" % (src, bookID))
# noinspection PyBroadException
try:
cachedir = lazylibrarian.CACHEDIR
coverfile = os.path.join(cachedir, "book", bookID + '.jpg')
if not src or src == 'cache' or src == 'current':
if os.path.isfile(coverfile): # use cached image if there is one
lazylibrarian.CACHE_HIT = int(lazylibrarian.CACHE_HIT) + 1
coverlink = 'cache/book/' + bookID + '.jpg'
return coverlink, 'cache'
elif src:
lazylibrarian.CACHE_MISS = int(lazylibrarian.CACHE_MISS) + 1
return None, src
myDB = database.DBConnection()
if not src or src == 'cover':
item = myDB.match('select BookFile from books where bookID=?', (bookID,))
if item:
bookfile = item['BookFile']
if bookfile: # we may have a cover.jpg in the same folder
bookdir = os.path.dirname(bookfile)
coverimg = os.path.join(bookdir, "cover.jpg")
if os.path.isfile(coverimg):
if src:
coverfile = os.path.join(cachedir, "book", bookID + '_cover.jpg')
coverlink = 'cache/book/' + bookID + '_cover.jpg'
logger.debug("Caching cover.jpg for %s" % bookID)
else:
coverlink = 'cache/book/' + bookID + '.jpg'
logger.debug("Caching cover.jpg for %s" % coverfile)
_ = safe_copy(coverimg, coverfile)
return coverlink, src
if src:
logger.debug('No cover.jpg found for %s' % bookID)
return None, src
# see if librarything has a cover
if not src or src == 'librarything':
if lazylibrarian.CONFIG['LT_DEVKEY']:
cmd = 'select BookISBN from books where bookID=?'
item = myDB.match(cmd, (bookID,))
if item and item['BookISBN']:
img = 'https://www.librarything.com/devkey/%s/large/isbn/%s' % (
lazylibrarian.CONFIG['LT_DEVKEY'], item['BookISBN'])
if src:
coverlink, success, _ = cache_img("book", bookID + '_lt', img)
else:
coverlink, success, _ = cache_img("book", bookID, img, refresh=True)
# if librarything has no image they return a 1x1 gif
data = ''
coverfile = os.path.join(lazylibrarian.DATADIR, coverlink)
if os.path.isfile(coverfile):
with open(coverfile, 'rb') as f:
data = f.read()
if len(data) < 50:
logger.debug('Got an empty librarything image for %s [%s]' % (bookID, coverlink))
elif success:
logger.debug("Caching librarything cover for %s" % bookID)
return coverlink, 'librarything'
else:
logger.debug('Failed to cache image for %s [%s]' % (img, coverlink))
else:
logger.debug("No isbn for %s" % bookID)
if src:
return None, src
# see if librarything workpage has a cover
if not src or src == 'whatwork':
work = getBookWork(bookID, "Cover")
if work:
try:
img = work.split('workCoverImage')[1].split('="')[1].split('"')[0]
if img and img.startswith('http'):
if src:
coverlink, success, _ = cache_img("book", bookID + '_ww', img)
else:
coverlink, success, _ = cache_img("book", bookID, img, refresh=True)
# if librarything has no image they return a 1x1 gif
data = ''
coverfile = os.path.join(lazylibrarian.DATADIR, coverlink)
if os.path.isfile(coverfile):
with open(coverfile, 'rb') as f:
data = f.read()
if len(data) < 50:
logger.debug('Got an empty whatwork image for %s [%s]' % (bookID, coverlink))
elif success:
logger.debug("Caching whatwork cover for %s" % bookID)
return coverlink, 'whatwork'
else:
logger.debug('Failed to cache image for %s [%s]' % (img, coverlink))
else:
logger.debug("No image found in work page for %s" % bookID)
except IndexError:
logger.debug('workCoverImage not found in work page for %s' % bookID)
try:
img = work.split('og:image')[1].split('="')[1].split('"')[0]
if img and img.startswith('http'):
if src:
coverlink, success, _ = cache_img("book", bookID + '_ww', img)
else:
coverlink, success, _ = cache_img("book", bookID, img, refresh=True)
# if librarything has no image they return a 1x1 gif
data = ''
coverfile = os.path.join(lazylibrarian.DATADIR, coverlink)
if os.path.isfile(coverfile):
with open(coverfile, 'rb') as f:
data = f.read()
if len(data) < 50:
logger.debug('Got an empty whatwork image for %s [%s]' % (bookID, coverlink))
if success:
logger.debug("Caching whatwork cover for %s" % bookID)
return coverlink, 'whatwork'
else:
logger.debug('Failed to cache image for %s [%s]' % (img, coverlink))
else:
logger.debug("No image found in work page for %s" % bookID)
except IndexError:
logger.debug('og:image not found in work page for %s' % bookID)
else:
logger.debug('No work page for %s' % bookID)
if src:
return None, src
cmd = 'select BookName,AuthorName,BookLink from books,authors where bookID=?'
cmd += ' and books.AuthorID = authors.AuthorID'
item = myDB.match(cmd, (bookID,))
safeparams = ''
booklink = ''
if item:
title = safe_unicode(item['BookName'])
author = safe_unicode(item['AuthorName'])
if PY2:
title = title.encode(lazylibrarian.SYS_ENCODING)
author = author.encode(lazylibrarian.SYS_ENCODING)
booklink = item['BookLink']
safeparams = quote_plus("%s %s" % (author, title))
# try to get a cover from goodreads
if not src or src == 'goodreads':
if booklink and 'goodreads' in booklink:
# if the bookID is a goodreads one, we can call https://www.goodreads.com/book/show/{bookID}
# and scrape the page for og:image
# <meta property="og:image" content="https://i.gr-assets.com/images/S/photo.goodreads.com/books/
# 1388267702i/16304._UY475_SS475_.jpg"/>
# to get the cover
result, success = fetchURL(booklink)
if success:
try:
img = result.split('id="coverImage"')[1].split('src="')[1].split('"')[0]
except IndexError:
try:
img = result.split('og:image')[1].split('="')[1].split('"')[0]
except IndexError:
img = None
if img and img.startswith('http') and 'nocover' not in img and 'nophoto' not in img:
if src == 'goodreads':
coverlink, success, _ = cache_img("book", bookID + '_gr', img)
else:
coverlink, success, _ = cache_img("book", bookID, img, refresh=True)
data = ''
coverfile = os.path.join(lazylibrarian.DATADIR, coverlink)
if os.path.isfile(coverfile):
with open(coverfile, 'rb') as f:
data = f.read()
if len(data) < 50:
logger.debug('Got an empty goodreads image for %s [%s]' % (bookID, coverlink))
elif success:
logger.debug("Caching goodreads cover for %s %s" % (item['AuthorName'], item['BookName']))
return coverlink, 'goodreads'
else:
logger.debug("Error getting goodreads image for %s, [%s]" % (img, coverlink))
else:
logger.debug("No image found in goodreads page for %s" % bookID)
else:
logger.debug("Error getting goodreads page %s, [%s]" % (booklink, result))
if src:
return None, src
if not src or src == 'googleisbn':
# try a google isbn page search...
# there is no image returned if google doesn't have a link for buying the book
if safeparams:
URL = "http://www.google.com/search?q=ISBN+" + safeparams
result, success = fetchURL(URL)
if success:
try:
img = result.split('imgurl=')[1].split('&imgrefurl')[0]
except IndexError:
try:
img = result.split('img src="')[1].split('"')[0]
except IndexError:
img = None
if img and img.startswith('http'):
if src:
coverlink, success, _ = cache_img("book", bookID + '_gi', img)
else:
coverlink, success, _ = cache_img("book", bookID, img, refresh=True)
data = ''
coverfile = os.path.join(lazylibrarian.DATADIR, coverlink)
if os.path.isfile(coverfile):
with open(coverfile, 'rb') as f:
data = f.read()
if len(data) < 50:
logger.debug('Got an empty google image for %s [%s]' % (bookID, coverlink))
elif success:
logger.debug("Caching google isbn cover for %s %s" %
(item['AuthorName'], item['BookName']))
return coverlink, 'google isbn'
else:
logger.debug("Error caching google image %s, [%s]" % (img, coverlink))
else:
logger.debug("No image found in google isbn page for %s" % bookID)
else:
logger.debug("Failed to fetch url from google")
else:
logger.debug("No parameters for google isbn search for %s" % bookID)
if src:
return None, src
if src == 'googleimage' or not src and lazylibrarian.CONFIG['IMP_GOOGLEIMAGE']:
# try a google image search...
# tbm=isch search images
# tbs=isz:l large images
# ift:jpg jpeg file type
if safeparams:
URL = "https://www.google.com/search?tbm=isch&tbs=isz:l,ift:jpg&as_q=" + safeparams + "+ebook"
img = None
result, success = fetchURL(URL)
if success:
try:
img = result.split('url?q=')[1].split('">')[1].split('src="')[1].split('"')[0]
except IndexError:
img = None
if img and img.startswith('http'):
if src:
coverlink, success, _ = cache_img("book", bookID + '_gb', img)
else:
coverlink, success, _ = cache_img("book", bookID, img, refresh=True)
data = ''
coverfile = os.path.join(lazylibrarian.DATADIR, coverlink)
if os.path.isfile(coverfile):
with open(coverfile, 'rb') as f:
data = f.read()
if len(data) < 50:
logger.debug('Got an empty goodreads image for %s [%s]' % (bookID, coverlink))
elif success:
logger.debug("Caching google search cover for %s %s" %
(item['AuthorName'], item['BookName']))
return coverlink, 'google image'
else:
logger.debug("Error getting google image %s, [%s]" % (img, coverlink))
else:
logger.debug("No image found in google page for %s" % bookID)
else:
logger.debug("No parameters for google image search for %s" % bookID)
if src:
return None, src
logger.debug("No image found from any configured source")
return None, src
except Exception:
logger.error('Unhandled exception in getBookCover: %s' % traceback.format_exc())
return None, src
def get_author_books(self, authorid=None, authorname=None, bookstatus="Skipped",
entrystatus='Active', refresh=False):
# noinspection PyBroadException
try:
logger.debug('[%s] Now processing books with Google Books API' % authorname)
# google doesnt like accents in author names
set_url = self.url + quote('inauthor:"%s"' % unaccented_str(authorname))
api_hits = 0
gr_lang_hits = 0
lt_lang_hits = 0
gb_lang_change = 0
cache_hits = 0
not_cached = 0
startindex = 0
resultcount = 0
removedResults = 0
duplicates = 0
ignored = 0
added_count = 0
updated_count = 0
book_ignore_count = 0
total_count = 0
number_results = 1
valid_langs = getList(lazylibrarian.CONFIG['IMP_PREFLANG'])
# Artist is loading
myDB = database.DBConnection()
controlValueDict = {"AuthorID": authorid}
newValueDict = {"Status": "Loading"}
myDB.upsert("authors", newValueDict, controlValueDict)
try:
while startindex < number_results:
self.params['startIndex'] = startindex
URL = set_url + '&' + urlencode(self.params)
try:
jsonresults, in_cache = gb_json_request(URL, useCache=not refresh)
if jsonresults is None:
number_results = 0
else:
if not in_cache:
api_hits += 1
number_results = jsonresults['totalItems']
except Exception as err:
if hasattr(err, 'reason'):
errmsg = err.reason
else:
errmsg = str(err)
logger.warn('Google Books API Error [%s]: Check your API key or wait a while' % errmsg)
break
if number_results == 0:
logger.warn('Found no results for %s' % authorname)
break
else:
logger.debug('Found %s result%s for %s' % (number_results, plural(number_results), authorname))
startindex += 40
for item in jsonresults['items']:
total_count += 1
book = bookdict(item)
# skip if no author, no author is no book.
if not book['author']:
logger.debug('Skipped a result without authorfield.')
continue
isbnhead = ""
if len(book['isbn']) == 10:
isbnhead = book['isbn'][0:3]
elif len(book['isbn']) == 13:
isbnhead = book['isbn'][3:6]
booklang = book['lang']
# do we care about language?
if "All" not in valid_langs:
if book['isbn']:
# seems google lies to us, sometimes tells us books are in english when they are not
if booklang == "Unknown" or booklang == "en":
googlelang = booklang
match = False
lang = myDB.match('SELECT lang FROM languages where isbn=?', (isbnhead,))
if lang:
booklang = lang['lang']
cache_hits += 1
logger.debug("Found cached language [%s] for [%s]" % (booklang, isbnhead))
match = True
if not match: # no match in cache, try lookup dict
if isbnhead:
if len(book['isbn']) == 13 and book['isbn'].startswith('979'):
for lang in lazylibrarian.isbn_979_dict:
if isbnhead.startswith(lang):
booklang = lazylibrarian.isbn_979_dict[lang]
logger.debug("ISBN979 returned %s for %s" %
(booklang, isbnhead))
match = True
break
elif (len(book['isbn']) == 10) or \
(len(book['isbn']) == 13 and book['isbn'].startswith('978')):
for lang in lazylibrarian.isbn_978_dict:
if isbnhead.startswith(lang):
booklang = lazylibrarian.isbn_978_dict[lang]
logger.debug("ISBN979 returned %s for %s" %
(booklang, isbnhead))
match = True
break
if match:
myDB.action('insert into languages values (?, ?)',
(isbnhead, booklang))
if not match:
booklang = thingLang(book['isbn'])
lt_lang_hits += 1
if booklang:
match = True
myDB.action('insert into languages values (?, ?)', (isbnhead, booklang))
if match:
# We found a better language match
if googlelang == "en" and booklang not in ["en-US", "en-GB", "eng"]:
# these are all english, may need to expand this list
logger.debug("%s Google thinks [%s], we think [%s]" %
(book['name'], googlelang, booklang))
gb_lang_change += 1
else: # No match anywhere, accept google language
booklang = googlelang
# skip if language is in ignore list
if booklang not in valid_langs:
logger.debug('Skipped [%s] with language %s' % (book['name'], booklang))
ignored += 1
continue
rejected = 0
check_status = False
book_status = bookstatus # new_book status, or new_author status
audio_status = lazylibrarian.CONFIG['NEWAUDIO_STATUS']
added = today()
locked = False
existing_book = None
bookname = book['name']
bookid = item['id']
if not bookname:
logger.debug('Rejecting bookid %s for %s, no bookname' % (bookid, authorname))
removedResults += 1
rejected = 1
else:
bookname = replace_all(unaccented(bookname), {':': '.', '"': '', '\'': ''}).strip()
# GoodReads sometimes has multiple bookids for the same book (same author/title, different
# editions) and sometimes uses the same bookid if the book is the same but the title is
# slightly different. Not sure if googlebooks does too, but we only want one...
cmd = 'SELECT Status,AudioStatus,Manual,BookAdded FROM books WHERE BookID=?'
existing_book = myDB.match(cmd, (bookid,))
if existing_book:
book_status = existing_book['Status']
audio_status = existing_book['AudioStatus']
locked = existing_book['Manual']
added = existing_book['BookAdded']
if locked is None:
locked = False
elif locked.isdigit():
locked = bool(int(locked))
else:
if rejected in [3, 4, 5]:
book_status = 'Ignored'
audio_status = 'Ignored'
else:
book_status = bookstatus # new_book status, or new_author status
audio_status = lazylibrarian.CONFIG['NEWAUDIO_STATUS']
added = today()
locked = False
if not rejected and re.match('[^\w-]', bookname): # remove books with bad characters in title
logger.debug("[%s] removed book for bad characters" % bookname)
removedResults += 1
rejected = 2
if not rejected and lazylibrarian.CONFIG['NO_FUTURE']:
# googlebooks sometimes gives yyyy, sometimes yyyy-mm, sometimes yyyy-mm-dd
if book['date'] > today()[:len(book['date'])]:
logger.debug('Rejecting %s, future publication date %s' % (bookname, book['date']))
removedResults += 1
rejected = 3
if not rejected and lazylibrarian.CONFIG['NO_PUBDATE']:
if not book['date']:
logger.debug('Rejecting %s, no publication date' % bookname)
removedResults += 1
rejected = 4
if not rejected and lazylibrarian.CONFIG['NO_ISBN']:
if not isbnhead:
logger.debug('Rejecting %s, no isbn' % bookname)
removedResults += 1
rejected = 5
if not rejected:
cmd = 'SELECT BookID FROM books,authors WHERE books.AuthorID = authors.AuthorID'
cmd += ' and BookName=? COLLATE NOCASE and AuthorName=? COLLATE NOCASE'
match = myDB.match(cmd, (bookname.replace('"', '""'), authorname.replace('"', '""')))
if match:
if match['BookID'] != bookid: # we have a different book with this author/title already
logger.debug('Rejecting bookid %s for [%s][%s] already got %s' %
(match['BookID'], authorname, bookname, bookid))
rejected = 6
duplicates += 1
if not rejected:
cmd = 'SELECT AuthorName,BookName FROM books,authors'
cmd += ' WHERE authors.AuthorID = books.AuthorID AND BookID=?'
match = myDB.match(cmd, (bookid,))
if match: # we have a book with this bookid already
if bookname != match['BookName'] or authorname != match['AuthorName']:
logger.debug('Rejecting bookid %s for [%s][%s] already got bookid for [%s][%s]' %
(bookid, authorname, bookname, match['AuthorName'], match['BookName']))
else:
logger.debug('Rejecting bookid %s for [%s][%s] already got this book in database' %
(bookid, authorname, bookname))
check_status = True
duplicates += 1
rejected = 7
if check_status or not rejected or (
lazylibrarian.CONFIG['IMP_IGNORE'] and rejected in [3, 4, 5]): # dates, isbn
if not locked:
controlValueDict = {"BookID": bookid}
newValueDict = {
"AuthorID": authorid,
"BookName": bookname,
"BookSub": book['sub'],
"BookDesc": book['desc'],
"BookIsbn": book['isbn'],
"BookPub": book['pub'],
"BookGenre": book['genre'],
"BookImg": book['img'],
"BookLink": book['link'],
"BookRate": float(book['rate']),
"BookPages": book['pages'],
"BookDate": book['date'],
"BookLang": booklang,
"Status": book_status,
"AudioStatus": audio_status,
"BookAdded": added
}
resultcount += 1
myDB.upsert("books", newValueDict, controlValueDict)
logger.debug("Book found: " + bookname + " " + book['date'])
updated = False
if 'nocover' in book['img'] or 'nophoto' in book['img']:
# try to get a cover from another source
workcover, source = getBookCover(bookid)
if workcover:
logger.debug('Updated cover for %s using %s' % (bookname, source))
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": workcover}
myDB.upsert("books", newValueDict, controlValueDict)
updated = True
elif book['img'] and book['img'].startswith('http'):
link, success, _ = cache_img("book", bookid, book['img'], refresh=refresh)
if success:
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": link}
myDB.upsert("books", newValueDict, controlValueDict)
updated = True
else:
logger.debug('Failed to cache image for %s' % book['img'])
serieslist = []
if book['series']:
serieslist = [('', book['seriesNum'], cleanName(unaccented(book['series']), '&/'))]
if lazylibrarian.CONFIG['ADD_SERIES']:
newserieslist = getWorkSeries(bookid)
if newserieslist:
serieslist = newserieslist
logger.debug('Updated series: %s [%s]' % (bookid, serieslist))
updated = True
setSeries(serieslist, bookid)
new_status = setStatus(bookid, serieslist, bookstatus)
if not new_status == book_status:
book_status = new_status
updated = True
worklink = getWorkPage(bookid)
if worklink:
controlValueDict = {"BookID": bookid}
newValueDict = {"WorkPage": worklink}
myDB.upsert("books", newValueDict, controlValueDict)
if not existing_book:
logger.debug("[%s] Added book: %s [%s] status %s" %
(authorname, bookname, booklang, book_status))
added_count += 1
elif updated:
logger.debug("[%s] Updated book: %s [%s] status %s" %
(authorname, bookname, booklang, book_status))
updated_count += 1
else:
book_ignore_count += 1
except KeyError:
pass
deleteEmptySeries()
logger.debug('[%s] The Google Books API was hit %s time%s to populate book list' %
(authorname, api_hits, plural(api_hits)))
cmd = 'SELECT BookName, BookLink, BookDate, BookImg from books WHERE AuthorID=?'
cmd += ' AND Status != "Ignored" order by BookDate DESC'
lastbook = myDB.match(cmd, (authorid,))
if lastbook: # maybe there are no books [remaining] for this author
lastbookname = lastbook['BookName']
lastbooklink = lastbook['BookLink']
lastbookdate = lastbook['BookDate']
lastbookimg = lastbook['BookImg']
else:
lastbookname = ""
lastbooklink = ""
lastbookdate = ""
lastbookimg = ""
controlValueDict = {"AuthorID": authorid}
newValueDict = {
"Status": entrystatus,
"LastBook": lastbookname,
"LastLink": lastbooklink,
"LastDate": lastbookdate,
"LastBookImg": lastbookimg
}
myDB.upsert("authors", newValueDict, controlValueDict)
logger.debug("Found %s total book%s for author" % (total_count, plural(total_count)))
logger.debug("Removed %s unwanted language result%s for author" % (ignored, plural(ignored)))
logger.debug("Removed %s bad character or no-name result%s for author" %
(removedResults, plural(removedResults)))
logger.debug("Removed %s duplicate result%s for author" % (duplicates, plural(duplicates)))
logger.debug("Found %s book%s by author marked as Ignored" % (book_ignore_count, plural(book_ignore_count)))
logger.debug("Imported/Updated %s book%s for author" % (resultcount, plural(resultcount)))
myDB.action('insert into stats values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
(authorname.replace('"', '""'), api_hits, gr_lang_hits, lt_lang_hits, gb_lang_change,
cache_hits, ignored, removedResults, not_cached, duplicates))
if refresh:
logger.info("[%s] Book processing complete: Added %s book%s / Updated %s book%s" %
(authorname, added_count, plural(added_count), updated_count, plural(updated_count)))
else:
logger.info("[%s] Book processing complete: Added %s book%s to the database" %
(authorname, added_count, plural(added_count)))
except Exception:
logger.error('Unhandled exception in GB.get_author_books: %s' % traceback.format_exc())
def addAuthorToDB(authorname=None,
refresh=False,
authorid=None,
addbooks=True):
"""
Add an author to the database by name or id, and optionally get a list of all their books
If author already exists in database, refresh their details and optionally booklist
"""
threadname = threading.currentThread().name
if "Thread-" in threadname:
threading.currentThread().name = "AddAuthorToDB"
try:
myDB = database.DBConnection()
match = False
authorimg = ''
new_author = not refresh
entry_status = ''
if authorid:
dbauthor = myDB.match("SELECT * from authors WHERE AuthorID=?",
(authorid, ))
if not dbauthor:
authorname = 'unknown author'
logger.debug("Adding new author id %s to database" % authorid)
new_author = True
else:
entry_status = dbauthor['Status']
authorname = dbauthor['authorname']
logger.debug("Updating author %s " % authorname)
new_author = False
controlValueDict = {"AuthorID": authorid}
newValueDict = {"Status": "Loading"}
myDB.upsert("authors", newValueDict, controlValueDict)
GR = GoodReads(authorname)
author = GR.get_author_info(authorid=authorid)
if author:
authorname = author['authorname']
authorimg = author['authorimg']
controlValueDict = {"AuthorID": authorid}
newValueDict = {
"AuthorLink": author['authorlink'],
"DateAdded": today()
}
if not dbauthor or (dbauthor and not dbauthor['manual']):
newValueDict["AuthorName"] = author['authorname']
newValueDict["AuthorImg"] = author['authorimg']
newValueDict["AuthorBorn"] = author['authorborn']
newValueDict["AuthorDeath"] = author['authordeath']
myDB.upsert("authors", newValueDict, controlValueDict)
match = True
else:
logger.warn(u"Nothing found for %s" % authorid)
if not dbauthor:
myDB.action('DELETE from authors WHERE AuthorID=?',
(authorid, ))
if authorname and not match:
authorname = ' '.join(
authorname.split()) # ensure no extra whitespace
GR = GoodReads(authorname)
author = GR.find_author_id(refresh=refresh)
query = "SELECT * from authors WHERE AuthorName=?"
dbauthor = myDB.match(query, (authorname.replace("'", "''"), ))
if author and not dbauthor: # may have different name for same authorid (spelling?)
query = "SELECT * from authors WHERE AuthorID=?"
dbauthor = myDB.match(query, (author['authorid'], ))
authorname = dbauthor['AuthorName']
controlValueDict = {"AuthorName": authorname}
if not dbauthor:
newValueDict = {
"AuthorID": "0: %s" % authorname,
"Status": "Loading"
}
logger.debug("Now adding new author: %s to database" %
authorname)
entry_status = lazylibrarian.CONFIG['NEWAUTHOR_STATUS']
new_author = True
else:
newValueDict = {"Status": "Loading"}
logger.debug("Now updating author: %s" % authorname)
entry_status = dbauthor['Status']
new_author = False
myDB.upsert("authors", newValueDict, controlValueDict)
if author:
authorid = author['authorid']
authorimg = author['authorimg']
controlValueDict = {"AuthorName": authorname}
newValueDict = {
"AuthorID": author['authorid'],
"AuthorLink": author['authorlink'],
"DateAdded": today(),
"Status": "Loading"
}
if not dbauthor or (dbauthor and not dbauthor['manual']):
newValueDict["AuthorImg"] = author['authorimg']
newValueDict["AuthorBorn"] = author['authorborn']
newValueDict["AuthorDeath"] = author['authordeath']
myDB.upsert("authors", newValueDict, controlValueDict)
match = True
else:
logger.warn(u"Nothing found for %s" % authorname)
if not dbauthor:
myDB.action('DELETE from authors WHERE AuthorName=?',
(authorname, ))
return
if not match:
logger.error(
"AddAuthorToDB: No matching result for authorname or authorid")
return
# if author is set to manual, should we allow replacing 'nophoto' ?
new_img = False
match = myDB.match("SELECT Manual from authors WHERE AuthorID=?",
(authorid, ))
if not match or not match['Manual']:
if authorimg and 'nophoto' in authorimg:
newimg = getAuthorImage(authorid)
if newimg:
authorimg = newimg
new_img = True
# allow caching
if authorimg and authorimg.startswith('http'):
newimg, success = cache_img("author",
authorid,
authorimg,
refresh=refresh)
if success:
authorimg = newimg
new_img = True
else:
logger.debug('Failed to cache image for %s' % authorimg)
if new_img:
controlValueDict = {"AuthorID": authorid}
newValueDict = {"AuthorImg": authorimg}
myDB.upsert("authors", newValueDict, controlValueDict)
if addbooks:
# audiostatus = lazylibrarian.CONFIG['NEWAUDIO_STATUS']
if new_author:
bookstatus = lazylibrarian.CONFIG['NEWAUTHOR_STATUS']
else:
bookstatus = lazylibrarian.CONFIG['NEWBOOK_STATUS']
if entry_status not in ['Active', 'Wanted', 'Ignored', 'Paused']:
entry_status = 'Active' # default for invalid/unknown or "loading"
# process books
if lazylibrarian.CONFIG['BOOK_API'] == "GoogleBooks":
book_api = GoogleBooks()
book_api.get_author_books(authorid,
authorname,
bookstatus,
entrystatus=entry_status,
refresh=refresh)
elif lazylibrarian.CONFIG['BOOK_API'] == "GoodReads":
book_api = GoodReads(authorname)
book_api.get_author_books(authorid,
authorname,
bookstatus,
entrystatus=entry_status,
refresh=refresh)
# update totals works for existing authors only.
# New authors need their totals updating after libraryscan or import of books.
if not new_author:
update_totals(authorid)
if new_author and lazylibrarian.CONFIG['GR_FOLLOWNEW']:
res = grfollow(authorid, True)
if res.startswith('Unable'):
logger.warn(res)
try:
followid = res.split("followid=")[1]
logger.debug('%s marked followed' % authorname)
except IndexError:
followid = ''
myDB.action('UPDATE authors SET GRfollow=? WHERE AuthorID=?',
(followid, authorid))
else:
# if we're not loading any books, mark author as ignored
entry_status = 'Ignored'
controlValueDict = {"AuthorID": authorid}
newValueDict = {"Status": entry_status}
myDB.upsert("authors", newValueDict, controlValueDict)
msg = "[%s] Author update complete, status %s" % (authorname,
entry_status)
logger.info(msg)
return msg
except Exception:
msg = 'Unhandled exception in addAuthorToDB: %s' % traceback.format_exc(
)
logger.error(msg)
return msg
def get_author_books(self, authorid=None, authorname=None, bookstatus="Skipped", refresh=False):
try:
api_hits = 0
gr_lang_hits = 0
lt_lang_hits = 0
gb_lang_change = 0
cache_hits = 0
not_cached = 0
URL = 'http://www.goodreads.com/author/list/' + authorid + '.xml?' + urllib.urlencode(self.params)
# Artist is loading
myDB = database.DBConnection()
controlValueDict = {"AuthorID": authorid}
newValueDict = {"Status": "Loading"}
myDB.upsert("authors", newValueDict, controlValueDict)
try:
rootxml, in_cache = get_xml_request(URL, useCache=not refresh)
except Exception as e:
logger.error("Error fetching author books: %s" % str(e))
return
if rootxml is None:
logger.debug("Error requesting author books")
return
if not in_cache:
api_hits += 1
resultxml = rootxml.getiterator('book')
valid_langs = getList(lazylibrarian.CONFIG['IMP_PREFLANG'])
resultsCount = 0
removedResults = 0
duplicates = 0
ignored = 0
added_count = 0
updated_count = 0
book_ignore_count = 0
total_count = 0
if not len(resultxml):
logger.warn('[%s] No books found for author with ID: %s' % (authorname, authorid))
else:
logger.debug("[%s] Now processing books with GoodReads API" % authorname)
logger.debug(u"url " + URL)
authorNameResult = rootxml.find('./author/name').text
# Goodreads sometimes puts extra whitepase in the author names!
authorNameResult = ' '.join(authorNameResult.split())
logger.debug(u"GoodReads author name [%s]" % authorNameResult)
loopCount = 1
while resultxml:
for book in resultxml:
total_count += 1
if book.find('publication_year').text is None:
pubyear = "0000"
else:
pubyear = book.find('publication_year').text
try:
bookimg = book.find('image_url').text
if 'nocover' in bookimg:
bookimg = 'images/nocover.png'
except (KeyError, AttributeError):
bookimg = 'images/nocover.png'
bookLanguage = "Unknown"
find_field = "id"
isbn = ""
isbnhead = ""
if "All" not in valid_langs: # do we care about language
if book.find('isbn').text:
find_field = "isbn"
isbn = book.find('isbn').text
isbnhead = isbn[0:3]
else:
if book.find('isbn13').text:
find_field = "isbn13"
isbn = book.find('isbn13').text
isbnhead = isbn[3:6]
# Try to use shortcut of ISBN identifier codes described here...
# https://en.wikipedia.org/wiki/List_of_ISBN_identifier_groups
if isbnhead:
if find_field == "isbn13" and isbn.startswith('979'):
for item in lazylibrarian.isbn_979_dict:
if isbnhead.startswith(item):
bookLanguage = lazylibrarian.isbn_979_dict[item]
break
if bookLanguage != "Unknown":
logger.debug("ISBN979 returned %s for %s" % (bookLanguage, isbnhead))
elif (find_field == "isbn") or (find_field == "isbn13" and isbn.startswith('978')):
for item in lazylibrarian.isbn_978_dict:
if isbnhead.startswith(item):
bookLanguage = lazylibrarian.isbn_978_dict[item]
break
if bookLanguage != "Unknown":
logger.debug("ISBN978 returned %s for %s" % (bookLanguage, isbnhead))
if bookLanguage == "Unknown" and isbnhead:
# Nothing in the isbn dictionary, try any cached results
match = myDB.match('SELECT lang FROM languages where isbn = "%s"' % isbnhead)
if match:
bookLanguage = match['lang']
cache_hits += 1
logger.debug("Found cached language [%s] for %s [%s]" %
(bookLanguage, find_field, isbnhead))
else:
# no match in cache, try searching librarything for a language code using the isbn
# if no language found, librarything return value is "invalid" or "unknown"
# returns plain text, not xml
BOOK_URL = 'http://www.librarything.com/api/thingLang.php?isbn=' + isbn
try:
librarything_wait()
resp = urllib2.urlopen(BOOK_URL, timeout=30).read()
lt_lang_hits += 1
logger.debug("LibraryThing reports language [%s] for %s" % (resp, isbnhead))
if 'invalid' in resp or 'Unknown' in resp:
bookLanguage = "Unknown"
else:
bookLanguage = resp # found a language code
myDB.action('insert into languages values ("%s", "%s")' %
(isbnhead, bookLanguage))
logger.debug(u"LT language %s: %s" % (isbnhead, bookLanguage))
except Exception as e:
logger.error("Error finding LT language result for [%s], %s" % (isbn, str(e)))
if bookLanguage == "Unknown":
# still no earlier match, we'll have to search the goodreads api
try:
if book.find(find_field).text:
BOOK_URL = 'http://www.goodreads.com/book/show?id=' + \
book.find(find_field).text + '&' + urllib.urlencode(self.params)
logger.debug(u"Book URL: " + BOOK_URL)
time_now = int(time.time())
if time_now <= lazylibrarian.LAST_GOODREADS:
time.sleep(1)
bookLanguage = ""
try:
BOOK_rootxml, in_cache = get_xml_request(BOOK_URL)
if BOOK_rootxml is None:
logger.debug('Error requesting book language code')
else:
if not in_cache:
# only update last_goodreads if the result wasn't found in the cache
lazylibrarian.LAST_GOODREADS = time_now
try:
bookLanguage = BOOK_rootxml.find('./book/language_code').text
except Exception as e:
logger.debug("Error finding language_code in book xml: %s" % str(e))
except Exception as e:
logger.debug("Error getting book xml: %s" % str(e))
if not in_cache:
gr_lang_hits += 1
if not bookLanguage:
bookLanguage = "Unknown"
# At this point, give up?
# WhatWork on author/title doesn't give us a language.
# It might give us the "original language" of the book (but not always)
# and our copy might not be in the original language anyway
# eg "The Girl With the Dragon Tattoo" original language Swedish
# If we have an isbn, try WhatISBN to get alternatives
# in case any of them give us a language, but it seems if thinglang doesn't
# have a language for the first isbn code, it doesn't for any of the
# alternatives either
# Goodreads search results don't include the language. Although sometimes
# it's in the html page, it's not in the xml results
if isbnhead != "":
# if GR didn't give an isbn we can't cache it, just use language for this book
myDB.action('insert into languages values ("%s", "%s")' %
(isbnhead, bookLanguage))
logger.debug("GoodReads reports language [%s] for %s" %
(bookLanguage, isbnhead))
else:
not_cached += 1
logger.debug(u"GR language: " + bookLanguage)
else:
logger.debug("No %s provided for [%s]" % (find_field, book.find('title').text))
# continue
except Exception as e:
logger.debug(u"Goodreads language search failed: %s" % str(e))
if bookLanguage not in valid_langs:
logger.debug('Skipped %s with language %s' % (book.find('title').text, bookLanguage))
ignored += 1
continue
bookname = book.find('title').text
bookid = book.find('id').text
bookdesc = book.find('description').text
bookisbn = book.find('isbn').text
bookpub = book.find('publisher').text
booklink = book.find('link').text
bookrate = float(book.find('average_rating').text)
bookpages = book.find('num_pages').text
bookname = unaccented(bookname)
bookname, booksub = split_title(authorNameResult, bookname)
dic = {':': '.', '"': ''} # do we need to strip apostrophes , '\'': ''}
bookname = replace_all(bookname, dic)
bookname = bookname.strip() # strip whitespace
booksub = replace_all(booksub, dic)
booksub = booksub.strip() # strip whitespace
if booksub:
series, seriesNum = bookSeries(booksub)
else:
series, seriesNum = bookSeries(bookname)
rejected = False
check_status = False
if re.match('[^\w-]', bookname): # reject books with bad characters in title
logger.debug(u"removed result [" + bookname + "] for bad characters")
removedResults += 1
rejected = True
if not rejected and lazylibrarian.CONFIG['NO_FUTURE']:
if pubyear > today()[:4]:
logger.debug('Rejecting %s, future publication date %s' % (bookname, pubyear))
removedResults += 1
rejected = True
if not rejected and not bookname:
logger.debug('Rejecting bookid %s for %s, no bookname' %
(bookid, authorNameResult))
removedResults += 1
rejected = True
if not rejected:
cmd = 'SELECT BookID FROM books,authors WHERE books.AuthorID = authors.AuthorID'
cmd += ' and BookName = "%s" COLLATE NOCASE and AuthorName = "%s" COLLATE NOCASE' % \
(bookname, authorNameResult.replace('"', '""'))
match = myDB.match(cmd)
if match:
if match['BookID'] != bookid:
# we have a different book with this author/title already
logger.debug('Rejecting bookid %s for [%s][%s] already got %s' %
(match['BookID'], authorNameResult, bookname, bookid))
duplicates += 1
rejected = True
if not rejected:
cmd = 'SELECT AuthorName,BookName FROM books,authors'
cmd += ' WHERE authors.AuthorID = books.AuthorID AND BookID=%s' % bookid
match = myDB.match(cmd)
if match:
# we have a book with this bookid already
if bookname != match['BookName'] or authorNameResult != match['AuthorName']:
logger.debug('Rejecting bookid %s for [%s][%s] already got bookid for [%s][%s]' %
(bookid, authorNameResult, bookname,
match['AuthorName'], match['BookName']))
else:
logger.debug('Rejecting bookid %s for [%s][%s] already got this book in database' %
(bookid, authorNameResult, bookname))
check_status = True
duplicates += 1
rejected = True
if check_status or not rejected:
existing_book = myDB.match('SELECT Status,Manual FROM books WHERE BookID = "%s"' % bookid)
if existing_book:
book_status = existing_book['Status']
locked = existing_book['Manual']
if locked is None:
locked = False
elif locked.isdigit():
locked = bool(int(locked))
else:
book_status = bookstatus # new_book status, or new_author status
locked = False
# Is the book already in the database?
# Leave alone if locked or status "ignore"
if not locked and book_status != "Ignored":
controlValueDict = {"BookID": bookid}
newValueDict = {
"AuthorID": authorid,
"BookName": bookname,
"BookSub": booksub,
"BookDesc": bookdesc,
"BookIsbn": bookisbn,
"BookPub": bookpub,
"BookGenre": "",
"BookImg": bookimg,
"BookLink": booklink,
"BookRate": bookrate,
"BookPages": bookpages,
"BookDate": pubyear,
"BookLang": bookLanguage,
"Status": book_status,
"BookAdded": today()
}
resultsCount += 1
updated = False
myDB.upsert("books", newValueDict, controlValueDict)
logger.debug(u"Book found: " + book.find('title').text + " " + pubyear)
if 'nocover' in bookimg or 'nophoto' in bookimg:
# try to get a cover from librarything
workcover = getBookCover(bookid)
if workcover:
logger.debug(u'Updated cover for %s to %s' % (bookname, workcover))
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": workcover}
myDB.upsert("books", newValueDict, controlValueDict)
updated = True
elif bookimg and bookimg.startswith('http'):
link, success = cache_img("book", bookid, bookimg, refresh=refresh)
if success:
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": link}
myDB.upsert("books", newValueDict, controlValueDict)
updated = True
else:
logger.debug('Failed to cache image for %s' % bookimg)
seriesdict = {}
if lazylibrarian.CONFIG['ADD_SERIES']:
# prefer series info from librarything
seriesdict = getWorkSeries(bookid)
if seriesdict:
logger.debug(u'Updated series: %s [%s]' % (bookid, seriesdict))
updated = True
else:
if series:
seriesdict = {cleanName(unaccented(series)): seriesNum}
setSeries(seriesdict, bookid)
new_status = setStatus(bookid, seriesdict, bookstatus)
if not new_status == book_status:
book_status = new_status
updated = True
worklink = getWorkPage(bookid)
if worklink:
controlValueDict = {"BookID": bookid}
newValueDict = {"WorkPage": worklink}
myDB.upsert("books", newValueDict, controlValueDict)
if not existing_book:
logger.debug(u"[%s] Added book: %s [%s] status %s" %
(authorname, bookname, bookLanguage, book_status))
added_count += 1
elif updated:
logger.debug(u"[%s] Updated book: %s [%s] status %s" %
(authorname, bookname, bookLanguage, book_status))
updated_count += 1
else:
book_ignore_count += 1
loopCount += 1
URL = 'http://www.goodreads.com/author/list/' + authorid + '.xml?' + \
urllib.urlencode(self.params) + '&page=' + str(loopCount)
resultxml = None
try:
rootxml, in_cache = get_xml_request(URL, useCache=not refresh)
if rootxml is None:
logger.debug('Error requesting next page of results')
else:
resultxml = rootxml.getiterator('book')
if not in_cache:
api_hits += 1
except Exception as e:
resultxml = None
logger.error("Error finding next page of results: %s" % str(e))
if resultxml:
if all(False for _ in resultxml): # returns True if iterator is empty
resultxml = None
deleteEmptySeries()
lastbook = myDB.match('SELECT BookName, BookLink, BookDate, BookImg from books WHERE AuthorID="%s" \
AND Status != "Ignored" order by BookDate DESC' % authorid)
if lastbook:
lastbookname = lastbook['BookName']
lastbooklink = lastbook['BookLink']
lastbookdate = lastbook['BookDate']
lastbookimg = lastbook['BookImg']
else:
lastbookname = ""
lastbooklink = ""
lastbookdate = ""
lastbookimg = ""
controlValueDict = {"AuthorID": authorid}
newValueDict = {
"Status": "Active",
"LastBook": lastbookname,
"LastLink": lastbooklink,
"LastDate": lastbookdate,
"LastBookImg": lastbookimg
}
myDB.upsert("authors", newValueDict, controlValueDict)
# This is here because GoodReads sometimes has several entries with the same BookID!
modified_count = added_count + updated_count
logger.debug("Found %s result%s" % (total_count, plural(total_count)))
logger.debug("Removed %s unwanted language result%s" % (ignored, plural(ignored)))
logger.debug(
"Removed %s bad character or no-name result%s" %
(removedResults, plural(removedResults)))
logger.debug("Removed %s duplicate result%s" % (duplicates, plural(duplicates)))
logger.debug("Found %s book%s by author marked as Ignored" % (book_ignore_count, plural(book_ignore_count)))
logger.debug("Imported/Updated %s book%s" % (modified_count, plural(modified_count)))
myDB.action('insert into stats values ("%s", %i, %i, %i, %i, %i, %i, %i, %i, %i)' %
(authorname.replace('"', '""'), api_hits, gr_lang_hits, lt_lang_hits, gb_lang_change,
cache_hits, ignored, removedResults, not_cached, duplicates))
if refresh:
logger.info("[%s] Book processing complete: Added %s book%s / Updated %s book%s" %
(authorname, added_count, plural(added_count), updated_count, plural(updated_count)))
else:
logger.info("[%s] Book processing complete: Added %s book%s to the database" %
(authorname, added_count, plural(added_count)))
except Exception:
logger.error('Unhandled exception in GR.get_author_books: %s' % traceback.format_exc())
def addAuthorToDB(authorname=None,
refresh=False,
authorid=None,
addbooks=True):
"""
Add an author to the database by name or id, and optionally get a list of all their books
If author already exists in database, refresh their details and optionally booklist
"""
threadname = threading.currentThread().name
if "Thread-" in threadname:
threading.currentThread().name = "AddAuthorToDB"
try:
myDB = database.DBConnection()
match = False
authorimg = ''
new_author = not refresh
if authorid:
controlValueDict = {"AuthorID": authorid}
newValueDict = {"Status": "Loading"}
dbauthor = myDB.match("SELECT * from authors WHERE AuthorID='%s'" %
authorid)
if not dbauthor:
authorname = 'unknown author'
logger.debug("Now adding new author id: %s to database" %
authorid)
new_author = True
else:
authorname = dbauthor['authorname']
logger.debug("Now updating author %s " % authorname)
new_author = False
myDB.upsert("authors", newValueDict, controlValueDict)
GR = GoodReads(authorname)
author = GR.get_author_info(authorid=authorid)
if author:
authorname = author['authorname']
authorimg = author['authorimg']
controlValueDict = {"AuthorID": authorid}
newValueDict = {
"AuthorLink": author['authorlink'],
"DateAdded": today()
}
if not dbauthor or (dbauthor and not dbauthor['manual']):
newValueDict["AuthorName"] = author['authorname']
newValueDict["AuthorImg"] = author['authorimg']
newValueDict["AuthorBorn"] = author['authorborn']
newValueDict["AuthorDeath"] = author['authordeath']
myDB.upsert("authors", newValueDict, controlValueDict)
match = True
else:
logger.warn(u"Nothing found for %s" % authorid)
if not dbauthor:
myDB.action('DELETE from authors WHERE AuthorID="%s"' %
authorid)
if authorname and not match:
authorname = ' '.join(
authorname.split()) # ensure no extra whitespace
GR = GoodReads(authorname)
query = "SELECT * from authors WHERE AuthorName='%s'" % authorname.replace(
"'", "''")
dbauthor = myDB.match(query)
controlValueDict = {"AuthorName": authorname}
if not dbauthor:
newValueDict = {
"AuthorID": "0: %s" % authorname,
"Status": "Loading"
}
logger.debug("Now adding new author: %s to database" %
authorname)
new_author = True
else:
newValueDict = {"Status": "Loading"}
logger.debug("Now updating author: %s" % authorname)
new_author = False
myDB.upsert("authors", newValueDict, controlValueDict)
author = GR.find_author_id(refresh=refresh)
if author:
authorid = author['authorid']
authorimg = author['authorimg']
controlValueDict = {"AuthorName": authorname}
newValueDict = {
"AuthorID": author['authorid'],
"AuthorLink": author['authorlink'],
"DateAdded": today(),
"Status": "Loading"
}
if not dbauthor or (dbauthor and not dbauthor['manual']):
newValueDict["AuthorImg"] = author['authorimg']
newValueDict["AuthorBorn"] = author['authorborn']
newValueDict["AuthorDeath"] = author['authordeath']
myDB.upsert("authors", newValueDict, controlValueDict)
match = True
else:
logger.warn(u"Nothing found for %s" % authorname)
if not dbauthor:
myDB.action('DELETE from authors WHERE AuthorName="%s"' %
authorname)
return
if not match:
logger.error(
"AddAuthorToDB: No matching result for authorname or authorid")
return
# if author is set to manual, should we allow replacing 'nophoto' ?
new_img = False
match = myDB.match("SELECT Manual from authors WHERE AuthorID='%s'" %
authorid)
if not match or not match['Manual']:
if authorimg and 'nophoto' in authorimg:
newimg = getAuthorImage(authorid)
if newimg:
authorimg = newimg
new_img = True
# allow caching
if authorimg and authorimg.startswith('http'):
newimg, success = cache_img("author",
authorid,
authorimg,
refresh=refresh)
if success:
authorimg = newimg
new_img = True
else:
logger.debug('Failed to cache image for %s' % authorimg)
if new_img:
controlValueDict = {"AuthorID": authorid}
newValueDict = {"AuthorImg": authorimg}
myDB.upsert("authors", newValueDict, controlValueDict)
if addbooks:
if new_author:
bookstatus = lazylibrarian.CONFIG['NEWAUTHOR_STATUS']
else:
bookstatus = lazylibrarian.CONFIG['NEWBOOK_STATUS']
# process books
if lazylibrarian.CONFIG['BOOK_API'] == "GoogleBooks":
book_api = GoogleBooks()
book_api.get_author_books(authorid,
authorname,
bookstatus,
refresh=refresh)
elif lazylibrarian.CONFIG['BOOK_API'] == "GoodReads":
GR = GoodReads(authorname)
GR.get_author_books(authorid,
authorname,
bookstatus,
refresh=refresh)
# update totals works for existing authors only.
# New authors need their totals updating after libraryscan or import of books.
if not new_author:
update_totals(authorid)
else:
# if we're not loading any books, mark author as ignored
controlValueDict = {"AuthorID": authorid}
newValueDict = {"Status": "Ignored"}
myDB.upsert("authors", newValueDict, controlValueDict)
msg = "[%s] Author update complete" % authorname
logger.debug(msg)
return msg
except Exception:
msg = 'Unhandled exception in addAuthorToDB: %s' % traceback.format_exc(
)
logger.error(msg)
return msg
def find_book(self, bookid=None, queue=None):
myDB = database.DBConnection()
URL = 'https://www.goodreads.com/book/show/' + bookid + '?' + urllib.urlencode(self.params)
try:
rootxml, in_cache = get_xml_request(URL)
if rootxml is None:
logger.debug("Error requesting book")
return
except Exception as e:
logger.error("Error finding book: %s" % str(e))
return
bookLanguage = rootxml.find('./book/language_code').text
bookname = rootxml.find('./book/title').text
if not bookLanguage:
bookLanguage = "Unknown"
#
# PAB user has said they want this book, don't block for unwanted language, just warn
#
valid_langs = getList(lazylibrarian.CONFIG['IMP_PREFLANG'])
if bookLanguage not in valid_langs:
logger.debug('Book %s goodreads language does not match preference, %s' % (bookname, bookLanguage))
if rootxml.find('./book/publication_year').text is None:
bookdate = "0000"
else:
bookdate = rootxml.find('./book/publication_year').text
try:
bookimg = rootxml.find('./book/img_url').text
if 'assets/nocover' in bookimg:
bookimg = 'images/nocover.png'
except (KeyError, AttributeError):
bookimg = 'images/nocover.png'
authorname = rootxml.find('./book/authors/author/name').text
bookdesc = rootxml.find('./book/description').text
bookisbn = rootxml.find('./book/isbn').text
bookpub = rootxml.find('./book/publisher').text
booklink = rootxml.find('./book/link').text
bookrate = float(rootxml.find('./book/average_rating').text)
bookpages = rootxml.find('.book/num_pages').text
name = authorname
GR = GoodReads(name)
author = GR.find_author_id()
if author:
AuthorID = author['authorid']
match = myDB.match('SELECT AuthorID from authors WHERE AuthorID="%s"' % AuthorID)
if not match:
match = myDB.match('SELECT AuthorID from authors WHERE AuthorName="%s"' % author['authorname'])
if match:
logger.debug('%s: Changing authorid from %s to %s' %
(author['authorname'], AuthorID, match['AuthorID']))
AuthorID = match['AuthorID'] # we have a different authorid for that authorname
else: # no author but request to add book, add author as "ignored"
# User hit "add book" button from a search
controlValueDict = {"AuthorID": AuthorID}
newValueDict = {
"AuthorName": author['authorname'],
"AuthorImg": author['authorimg'],
"AuthorLink": author['authorlink'],
"AuthorBorn": author['authorborn'],
"AuthorDeath": author['authordeath'],
"DateAdded": today(),
"Status": "Ignored"
}
myDB.upsert("authors", newValueDict, controlValueDict)
else:
logger.warn("No AuthorID for %s, unable to add book %s" % (authorname, bookname))
return
bookname = unaccented(bookname)
bookname, booksub = split_title(authorname, bookname)
dic = {':': '.', '"': '', '\'': ''}
bookname = replace_all(bookname, dic).strip()
booksub = replace_all(booksub, dic).strip()
if booksub:
series, seriesNum = bookSeries(booksub)
else:
series, seriesNum = bookSeries(bookname)
controlValueDict = {"BookID": bookid}
newValueDict = {
"AuthorID": AuthorID,
"BookName": bookname,
"BookSub": booksub,
"BookDesc": bookdesc,
"BookIsbn": bookisbn,
"BookPub": bookpub,
"BookGenre": "",
"BookImg": bookimg,
"BookLink": booklink,
"BookRate": bookrate,
"BookPages": bookpages,
"BookDate": bookdate,
"BookLang": bookLanguage,
"Status": "Wanted",
"BookAdded": today()
}
myDB.upsert("books", newValueDict, controlValueDict)
logger.info("%s added to the books database" % bookname)
if 'nocover' in bookimg or 'nophoto' in bookimg:
# try to get a cover from librarything
workcover = getBookCover(bookid)
if workcover:
logger.debug(u'Updated cover for %s to %s' % (bookname, workcover))
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": workcover}
myDB.upsert("books", newValueDict, controlValueDict)
elif bookimg and bookimg.startswith('http'):
link, success = cache_img("book", bookid, bookimg)
if success:
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": link}
myDB.upsert("books", newValueDict, controlValueDict)
else:
logger.debug('Failed to cache image for %s' % bookimg)
if lazylibrarian.CONFIG['ADD_SERIES']:
# prefer series info from librarything
seriesdict = getWorkSeries(bookid)
if seriesdict:
logger.debug(u'Updated series: %s [%s]' % (bookid, seriesdict))
else:
if series:
seriesdict = {cleanName(unaccented(series)): seriesNum}
setSeries(seriesdict, bookid)
worklink = getWorkPage(bookid)
if worklink:
controlValueDict = {"BookID": bookid}
newValueDict = {"WorkPage": worklink}
myDB.upsert("books", newValueDict, controlValueDict)
def getBookCover(bookID=None, src=None):
""" Return link to a local file containing a book cover image for a bookid, and which source used.
Try 1. Local file cached from goodreads/googlebooks when book was imported
2. cover.jpg if we have the book
3. LibraryThing cover image (if you have a dev key)
4. LibraryThing whatwork (if available)
5. Goodreads search (if book was imported from goodreads)
6. OpenLibrary image
7. Google isbn search (if google has a link to book for sale)
8. Google images search (if lazylibrarian config allows)
src = cache, cover, goodreads, librarything, whatwork, googleisbn, openlibrary, googleimage
Return None if no cover available. """
if not bookID:
logger.error("getBookCover- No bookID")
return None, src
if not src:
src = ''
logger.debug("Getting %s cover for %s" % (src, bookID))
# noinspection PyBroadException
try:
cachedir = lazylibrarian.CACHEDIR
coverfile = os.path.join(cachedir, "book", bookID + '.jpg')
if not src or src == 'cache' or src == 'current':
if os.path.isfile(coverfile): # use cached image if there is one
lazylibrarian.CACHE_HIT = int(lazylibrarian.CACHE_HIT) + 1
coverlink = 'cache/book/' + bookID + '.jpg'
return coverlink, 'cache'
elif src:
lazylibrarian.CACHE_MISS = int(lazylibrarian.CACHE_MISS) + 1
return None, src
myDB = database.DBConnection()
if not src or src == 'cover':
item = myDB.match('select BookFile from books where bookID=?', (bookID,))
if item:
bookfile = item['BookFile']
if bookfile: # we may have a cover.jpg in the same folder
bookdir = os.path.dirname(bookfile)
coverimg = os.path.join(bookdir, "cover.jpg")
if os.path.isfile(coverimg):
if src:
coverfile = os.path.join(cachedir, "book", bookID + '_cover.jpg')
coverlink = 'cache/book/' + bookID + '_cover.jpg'
logger.debug("Caching cover.jpg for %s" % bookID)
else:
coverlink = 'cache/book/' + bookID + '.jpg'
logger.debug("Caching cover.jpg for %s" % coverfile)
_ = safe_copy(coverimg, coverfile)
return coverlink, src
if src:
logger.debug('No cover.jpg found for %s' % bookID)
return None, src
# see if librarything has a cover
if not src or src == 'librarything':
if lazylibrarian.CONFIG['LT_DEVKEY']:
cmd = 'select BookISBN from books where bookID=?'
item = myDB.match(cmd, (bookID,))
if item and item['BookISBN']:
img = 'https://www.librarything.com/devkey/%s/large/isbn/%s' % (
lazylibrarian.CONFIG['LT_DEVKEY'], item['BookISBN'])
if src:
coverlink, success, _ = cache_img("book", bookID + '_lt', img)
else:
coverlink, success, _ = cache_img("book", bookID, img, refresh=True)
# if librarything has no image they return a 1x1 gif
data = ''
coverfile = os.path.join(lazylibrarian.DATADIR, coverlink)
if os.path.isfile(coverfile):
with open(coverfile, 'rb') as f:
data = f.read()
if len(data) < 50:
logger.debug('Got an empty librarything image for %s [%s]' % (bookID, coverlink))
elif success:
logger.debug("Caching librarything cover for %s" % bookID)
return coverlink, 'librarything'
else:
logger.debug('Failed to cache image for %s [%s]' % (img, coverlink))
else:
logger.debug("No isbn for %s" % bookID)
if src:
return None, src
# see if librarything workpage has a cover
if not src or src == 'whatwork':
work = getBookWork(bookID, "Cover")
if work:
try:
img = work.split('workCoverImage')[1].split('="')[1].split('"')[0]
if img and img.startswith('http'):
if src:
coverlink, success, _ = cache_img("book", bookID + '_ww', img)
else:
coverlink, success, _ = cache_img("book", bookID, img, refresh=True)
# if librarything has no image they return a 1x1 gif
data = ''
coverfile = os.path.join(lazylibrarian.DATADIR, coverlink)
if os.path.isfile(coverfile):
with open(coverfile, 'rb') as f:
data = f.read()
if len(data) < 50:
logger.debug('Got an empty whatwork image for %s [%s]' % (bookID, coverlink))
elif success:
logger.debug("Caching whatwork cover for %s" % bookID)
return coverlink, 'whatwork'
else:
logger.debug('Failed to cache image for %s [%s]' % (img, coverlink))
else:
logger.debug("No image found in work page for %s" % bookID)
except IndexError:
logger.debug('workCoverImage not found in work page for %s' % bookID)
try:
img = work.split('og:image')[1].split('="')[1].split('"')[0]
if img and img.startswith('http'):
if src:
coverlink, success, _ = cache_img("book", bookID + '_ww', img)
else:
coverlink, success, _ = cache_img("book", bookID, img, refresh=True)
# if librarything has no image they return a 1x1 gif
data = ''
coverfile = os.path.join(lazylibrarian.DATADIR, coverlink)
if os.path.isfile(coverfile):
with open(coverfile, 'rb') as f:
data = f.read()
if len(data) < 50:
logger.debug('Got an empty whatwork image for %s [%s]' % (bookID, coverlink))
if success:
logger.debug("Caching whatwork cover for %s" % bookID)
return coverlink, 'whatwork'
else:
logger.debug('Failed to cache image for %s [%s]' % (img, coverlink))
else:
logger.debug("No image found in work page for %s" % bookID)
except IndexError:
logger.debug('og:image not found in work page for %s' % bookID)
else:
logger.debug('No work page for %s' % bookID)
if src:
return None, src
cmd = 'select BookName,AuthorName,BookLink,BookISBN from books,authors where bookID=?'
cmd += ' and books.AuthorID = authors.AuthorID'
item = myDB.match(cmd, (bookID,))
safeparams = ''
booklink = ''
if item:
title = safe_unicode(item['BookName'])
author = safe_unicode(item['AuthorName'])
if PY2:
title = title.encode(lazylibrarian.SYS_ENCODING)
author = author.encode(lazylibrarian.SYS_ENCODING)
booklink = item['BookLink']
safeparams = quote_plus("%s %s" % (author, title))
# try to get a cover from goodreads
if not src or src == 'goodreads':
if booklink and 'goodreads' in booklink:
# if the bookID is a goodreads one, we can call https://www.goodreads.com/book/show/{bookID}
# and scrape the page for og:image
# <meta property="og:image" content="https://i.gr-assets.com/images/S/photo.goodreads.com/books/
# 1388267702i/16304._UY475_SS475_.jpg"/>
# to get the cover
result, success = fetchURL(booklink)
if success:
try:
img = result.split('id="coverImage"')[1].split('src="')[1].split('"')[0]
except IndexError:
try:
img = result.split('og:image')[1].split('="')[1].split('"')[0]
except IndexError:
img = None
if img and img.startswith('http') and 'nocover' not in img and 'nophoto' not in img:
if src == 'goodreads':
coverlink, success, _ = cache_img("book", bookID + '_gr', img)
else:
coverlink, success, _ = cache_img("book", bookID, img, refresh=True)
data = ''
coverfile = os.path.join(lazylibrarian.DATADIR, coverlink)
if os.path.isfile(coverfile):
with open(coverfile, 'rb') as f:
data = f.read()
if len(data) < 50:
logger.debug('Got an empty goodreads image for %s [%s]' % (bookID, coverlink))
elif success:
logger.debug("Caching goodreads cover for %s %s" % (item['AuthorName'], item['BookName']))
return coverlink, 'goodreads'
else:
logger.debug("Error getting goodreads image for %s, [%s]" % (img, coverlink))
else:
logger.debug("No image found in goodreads page for %s" % bookID)
else:
logger.debug("Error getting goodreads page %s, [%s]" % (booklink, result))
if src:
return None, src
# try to get a cover from openlibrary
if not src or src == 'openlibrary':
if item['BookISBN']:
baseurl = 'https://openlibrary.org/api/books?format=json&jscmd=data&bibkeys=ISBN:'
result, success = fetchURL(baseurl + item['BookISBN'])
if success:
try:
source = json.loads(result) # type: dict
except Exception as e:
logger.debug("OpenLibrary json error: %s" % e)
source = []
img = ''
if source:
# noinspection PyUnresolvedReferences
k = source.keys()[0]
try:
img = source[k]['cover']['medium']
except KeyError:
try:
img = source[k]['cover']['large']
except KeyError:
logger.debug("No openlibrary image for %s" % item['BookISBN'])
if img and img.startswith('http') and 'nocover' not in img and 'nophoto' not in img:
if src == 'openlibrary':
coverlink, success, _ = cache_img("book", bookID + '_ol', img)
else:
coverlink, success, _ = cache_img("book", bookID, img, refresh=True)
data = ''
coverfile = os.path.join(lazylibrarian.DATADIR, coverlink)
if os.path.isfile(coverfile):
with open(coverfile, 'rb') as f:
data = f.read()
if len(data) < 50:
logger.debug('Got an empty openlibrary image for %s [%s]' % (bookID, coverlink))
elif success:
logger.debug("Caching openlibrary cover for %s %s" % (item['AuthorName'], item['BookName']))
return coverlink, 'openlibrary'
else:
logger.debug("OpenLibrary error: %s" % result)
if src:
return None, src
if not src or src == 'googleisbn':
# try a google isbn page search...
# there is no image returned if google doesn't have a link for buying the book
if safeparams:
URL = "http://www.google.com/search?q=ISBN+" + safeparams
result, success = fetchURL(URL)
if success:
try:
img = result.split('imgurl=')[1].split('&imgrefurl')[0]
except IndexError:
try:
img = result.split('img src="')[1].split('"')[0]
except IndexError:
img = None
if img and img.startswith('http'):
if src:
coverlink, success, _ = cache_img("book", bookID + '_gi', img)
else:
coverlink, success, _ = cache_img("book", bookID, img, refresh=True)
data = ''
coverfile = os.path.join(lazylibrarian.DATADIR, coverlink)
if os.path.isfile(coverfile):
with open(coverfile, 'rb') as f:
data = f.read()
if len(data) < 50:
logger.debug('Got an empty google image for %s [%s]' % (bookID, coverlink))
elif success:
logger.debug("Caching google isbn cover for %s %s" %
(item['AuthorName'], item['BookName']))
return coverlink, 'google isbn'
else:
logger.debug("Error caching google image %s, [%s]" % (img, coverlink))
else:
logger.debug("No image found in google isbn page for %s" % bookID)
else:
logger.debug("Failed to fetch url from google")
else:
logger.debug("No parameters for google isbn search for %s" % bookID)
if src:
return None, src
if src == 'googleimage' or not src and lazylibrarian.CONFIG['IMP_GOOGLEIMAGE']:
# try a google image search...
# tbm=isch search images
# tbs=isz:l large images
# ift:jpg jpeg file type
if safeparams:
URL = "https://www.google.com/search?tbm=isch&tbs=isz:l,ift:jpg&as_q=" + safeparams + "+ebook"
img = None
result, success = fetchURL(URL)
if success:
try:
img = result.split('url?q=')[1].split('">')[1].split('src="')[1].split('"')[0]
except IndexError:
img = None
if img and img.startswith('http'):
if src:
coverlink, success, _ = cache_img("book", bookID + '_gb', img)
else:
coverlink, success, _ = cache_img("book", bookID, img, refresh=True)
data = ''
coverfile = os.path.join(lazylibrarian.DATADIR, coverlink)
if os.path.isfile(coverfile):
with open(coverfile, 'rb') as f:
data = f.read()
if len(data) < 50:
logger.debug('Got an empty goodreads image for %s [%s]' % (bookID, coverlink))
elif success:
logger.debug("Caching google search cover for %s %s" %
(item['AuthorName'], item['BookName']))
return coverlink, 'google image'
else:
logger.debug("Error getting google image %s, [%s]" % (img, coverlink))
else:
logger.debug("No image found in google page for %s" % bookID)
else:
logger.debug("No parameters for google image search for %s" % bookID)
if src:
return None, src
logger.debug("No image found from any configured source")
return None, src
except Exception:
logger.error('Unhandled exception in getBookCover: %s' % traceback.format_exc())
return None, src
def get_author_books(self,
authorid=None,
authorname=None,
bookstatus="Skipped",
entrystatus='Active',
refresh=False):
# noinspection PyBroadException
try:
logger.debug('[%s] Now processing books with Google Books API' %
authorname)
# google doesnt like accents in author names
set_url = self.url + quote(
'inauthor:"%s"' % unaccented_str(authorname))
api_hits = 0
gr_lang_hits = 0
lt_lang_hits = 0
gb_lang_change = 0
cache_hits = 0
not_cached = 0
startindex = 0
resultcount = 0
removedResults = 0
duplicates = 0
ignored = 0
added_count = 0
updated_count = 0
book_ignore_count = 0
total_count = 0
number_results = 1
valid_langs = getList(lazylibrarian.CONFIG['IMP_PREFLANG'])
# Artist is loading
myDB = database.DBConnection()
controlValueDict = {"AuthorID": authorid}
newValueDict = {"Status": "Loading"}
myDB.upsert("authors", newValueDict, controlValueDict)
try:
while startindex < number_results:
self.params['startIndex'] = startindex
URL = set_url + '&' + urlencode(self.params)
try:
jsonresults, in_cache = gb_json_request(
URL, useCache=not refresh)
if jsonresults is None:
number_results = 0
else:
if not in_cache:
api_hits += 1
number_results = jsonresults['totalItems']
except Exception as err:
if hasattr(err, 'reason'):
errmsg = err.reason
else:
errmsg = str(err)
logger.warn(
'Google Books API Error [%s]: Check your API key or wait a while'
% errmsg)
break
if number_results == 0:
logger.warn('Found no results for %s' % authorname)
break
else:
logger.debug('Found %s result%s for %s' %
(number_results, plural(number_results),
authorname))
startindex += 40
for item in jsonresults['items']:
total_count += 1
book = bookdict(item)
# skip if no author, no author is no book.
if not book['author']:
logger.debug(
'Skipped a result without authorfield.')
continue
isbnhead = ""
if len(book['isbn']) == 10:
isbnhead = book['isbn'][0:3]
elif len(book['isbn']) == 13:
isbnhead = book['isbn'][3:6]
booklang = book['lang']
# do we care about language?
if "All" not in valid_langs:
if book['isbn']:
# seems google lies to us, sometimes tells us books are in english when they are not
if booklang == "Unknown" or booklang == "en":
googlelang = booklang
match = False
lang = myDB.match(
'SELECT lang FROM languages where isbn=?',
(isbnhead, ))
if lang:
booklang = lang['lang']
cache_hits += 1
logger.debug(
"Found cached language [%s] for [%s]"
% (booklang, isbnhead))
match = True
if not match: # no match in cache, try lookup dict
if isbnhead:
if len(
book['isbn']
) == 13 and book[
'isbn'].startswith('979'):
for lang in lazylibrarian.isbn_979_dict:
if isbnhead.startswith(
lang):
booklang = lazylibrarian.isbn_979_dict[
lang]
logger.debug(
"ISBN979 returned %s for %s"
% (booklang,
isbnhead))
match = True
break
elif (len(book['isbn']) == 10) or \
(len(book['isbn']) == 13 and book['isbn'].startswith('978')):
for lang in lazylibrarian.isbn_978_dict:
if isbnhead.startswith(
lang):
booklang = lazylibrarian.isbn_978_dict[
lang]
logger.debug(
"ISBN979 returned %s for %s"
% (booklang,
isbnhead))
match = True
break
if match:
myDB.action(
'insert into languages values (?, ?)',
(isbnhead, booklang))
if not match:
booklang = thingLang(book['isbn'])
lt_lang_hits += 1
if booklang:
match = True
myDB.action(
'insert into languages values (?, ?)',
(isbnhead, booklang))
if match:
# We found a better language match
if googlelang == "en" and booklang not in [
"en-US", "en-GB", "eng"
]:
# these are all english, may need to expand this list
logger.debug(
"%s Google thinks [%s], we think [%s]"
% (book['name'], googlelang,
booklang))
gb_lang_change += 1
else: # No match anywhere, accept google language
booklang = googlelang
# skip if language is in ignore list
if booklang not in valid_langs:
logger.debug('Skipped [%s] with language %s' %
(book['name'], booklang))
ignored += 1
continue
rejected = 0
check_status = False
book_status = bookstatus # new_book status, or new_author status
audio_status = lazylibrarian.CONFIG['NEWAUDIO_STATUS']
added = today()
locked = False
existing_book = None
bookname = book['name']
bookid = item['id']
if not bookname:
logger.debug(
'Rejecting bookid %s for %s, no bookname' %
(bookid, authorname))
removedResults += 1
rejected = 1
else:
bookname = replace_all(unaccented(bookname), {
':': '.',
'"': '',
'\'': ''
}).strip()
# GoodReads sometimes has multiple bookids for the same book (same author/title, different
# editions) and sometimes uses the same bookid if the book is the same but the title is
# slightly different. Not sure if googlebooks does too, but we only want one...
cmd = 'SELECT Status,AudioStatus,Manual,BookAdded FROM books WHERE BookID=?'
existing_book = myDB.match(cmd, (bookid, ))
if existing_book:
book_status = existing_book['Status']
audio_status = existing_book['AudioStatus']
locked = existing_book['Manual']
added = existing_book['BookAdded']
if locked is None:
locked = False
elif locked.isdigit():
locked = bool(int(locked))
else:
if rejected in [3, 4, 5]:
book_status = 'Ignored'
audio_status = 'Ignored'
else:
book_status = bookstatus # new_book status, or new_author status
audio_status = lazylibrarian.CONFIG[
'NEWAUDIO_STATUS']
added = today()
locked = False
if not rejected and re.match(
'[^\w-]', bookname
): # remove books with bad characters in title
logger.debug(
"[%s] removed book for bad characters" %
bookname)
removedResults += 1
rejected = 2
if not rejected and lazylibrarian.CONFIG['NO_FUTURE']:
# googlebooks sometimes gives yyyy, sometimes yyyy-mm, sometimes yyyy-mm-dd
if book['date'] > today()[:len(book['date'])]:
logger.debug(
'Rejecting %s, future publication date %s'
% (bookname, book['date']))
removedResults += 1
rejected = 3
if not rejected and lazylibrarian.CONFIG['NO_PUBDATE']:
if not book['date']:
logger.debug(
'Rejecting %s, no publication date' %
bookname)
removedResults += 1
rejected = 4
if not rejected and lazylibrarian.CONFIG['NO_ISBN']:
if not isbnhead:
logger.debug('Rejecting %s, no isbn' %
bookname)
removedResults += 1
rejected = 5
if not rejected:
cmd = 'SELECT BookID FROM books,authors WHERE books.AuthorID = authors.AuthorID'
cmd += ' and BookName=? COLLATE NOCASE and AuthorName=? COLLATE NOCASE'
match = myDB.match(cmd, (bookname.replace(
'"', '""'), authorname.replace('"', '""')))
if match:
if match['BookID'] != bookid: # we have a different book with this author/title already
logger.debug(
'Rejecting bookid %s for [%s][%s] already got %s'
% (match['BookID'], authorname,
bookname, bookid))
rejected = 6
duplicates += 1
if not rejected:
cmd = 'SELECT AuthorName,BookName FROM books,authors'
cmd += ' WHERE authors.AuthorID = books.AuthorID AND BookID=?'
match = myDB.match(cmd, (bookid, ))
if match: # we have a book with this bookid already
if bookname != match[
'BookName'] or authorname != match[
'AuthorName']:
logger.debug(
'Rejecting bookid %s for [%s][%s] already got bookid for [%s][%s]'
% (bookid, authorname, bookname,
match['AuthorName'],
match['BookName']))
else:
logger.debug(
'Rejecting bookid %s for [%s][%s] already got this book in database'
% (bookid, authorname, bookname))
check_status = True
duplicates += 1
rejected = 7
if check_status or not rejected or (
lazylibrarian.CONFIG['IMP_IGNORE']
and rejected in [3, 4, 5]): # dates, isbn
if not locked:
controlValueDict = {"BookID": bookid}
newValueDict = {
"AuthorID": authorid,
"BookName": bookname,
"BookSub": book['sub'],
"BookDesc": book['desc'],
"BookIsbn": book['isbn'],
"BookPub": book['pub'],
"BookGenre": book['genre'],
"BookImg": book['img'],
"BookLink": book['link'],
"BookRate": float(book['rate']),
"BookPages": book['pages'],
"BookDate": book['date'],
"BookLang": booklang,
"Status": book_status,
"AudioStatus": audio_status,
"BookAdded": added
}
resultcount += 1
myDB.upsert("books", newValueDict,
controlValueDict)
logger.debug("Book found: " + bookname + " " +
book['date'])
updated = False
if 'nocover' in book[
'img'] or 'nophoto' in book['img']:
# try to get a cover from another source
workcover, source = getBookCover(bookid)
if workcover:
logger.debug(
'Updated cover for %s using %s' %
(bookname, source))
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": workcover}
myDB.upsert("books", newValueDict,
controlValueDict)
updated = True
elif book['img'] and book['img'].startswith(
'http'):
link, success, _ = cache_img(
"book",
bookid,
book['img'],
refresh=refresh)
if success:
controlValueDict = {"BookID": bookid}
newValueDict = {"BookImg": link}
myDB.upsert("books", newValueDict,
controlValueDict)
updated = True
else:
logger.debug(
'Failed to cache image for %s' %
book['img'])
serieslist = []
if book['series']:
serieslist = [
('', book['seriesNum'],
cleanName(unaccented(book['series']),
'&/'))
]
if lazylibrarian.CONFIG['ADD_SERIES']:
newserieslist = getWorkSeries(bookid)
if newserieslist:
serieslist = newserieslist
logger.debug(
'Updated series: %s [%s]' %
(bookid, serieslist))
updated = True
setSeries(serieslist, bookid)
new_status = setStatus(bookid, serieslist,
bookstatus)
if not new_status == book_status:
book_status = new_status
updated = True
worklink = getWorkPage(bookid)
if worklink:
controlValueDict = {"BookID": bookid}
newValueDict = {"WorkPage": worklink}
myDB.upsert("books", newValueDict,
controlValueDict)
if not existing_book:
logger.debug(
"[%s] Added book: %s [%s] status %s" %
(authorname, bookname, booklang,
book_status))
added_count += 1
elif updated:
logger.debug(
"[%s] Updated book: %s [%s] status %s"
% (authorname, bookname, booklang,
book_status))
updated_count += 1
else:
book_ignore_count += 1
except KeyError:
pass
deleteEmptySeries()
logger.debug(
'[%s] The Google Books API was hit %s time%s to populate book list'
% (authorname, api_hits, plural(api_hits)))
cmd = 'SELECT BookName, BookLink, BookDate, BookImg from books WHERE AuthorID=?'
cmd += ' AND Status != "Ignored" order by BookDate DESC'
lastbook = myDB.match(cmd, (authorid, ))
if lastbook: # maybe there are no books [remaining] for this author
lastbookname = lastbook['BookName']
lastbooklink = lastbook['BookLink']
lastbookdate = lastbook['BookDate']
lastbookimg = lastbook['BookImg']
else:
lastbookname = ""
lastbooklink = ""
lastbookdate = ""
lastbookimg = ""
controlValueDict = {"AuthorID": authorid}
newValueDict = {
"Status": entrystatus,
"LastBook": lastbookname,
"LastLink": lastbooklink,
"LastDate": lastbookdate,
"LastBookImg": lastbookimg
}
myDB.upsert("authors", newValueDict, controlValueDict)
logger.debug("Found %s total book%s for author" %
(total_count, plural(total_count)))
logger.debug("Removed %s unwanted language result%s for author" %
(ignored, plural(ignored)))
logger.debug(
"Removed %s bad character or no-name result%s for author" %
(removedResults, plural(removedResults)))
logger.debug("Removed %s duplicate result%s for author" %
(duplicates, plural(duplicates)))
logger.debug("Found %s book%s by author marked as Ignored" %
(book_ignore_count, plural(book_ignore_count)))
logger.debug("Imported/Updated %s book%s for author" %
(resultcount, plural(resultcount)))
myDB.action(
'insert into stats values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
(authorname.replace('"', '""'), api_hits, gr_lang_hits,
lt_lang_hits, gb_lang_change, cache_hits, ignored,
removedResults, not_cached, duplicates))
if refresh:
logger.info(
"[%s] Book processing complete: Added %s book%s / Updated %s book%s"
% (authorname, added_count, plural(added_count),
updated_count, plural(updated_count)))
else:
logger.info(
"[%s] Book processing complete: Added %s book%s to the database"
% (authorname, added_count, plural(added_count)))
except Exception:
logger.error('Unhandled exception in GB.get_author_books: %s' %
traceback.format_exc())
