def build_bookstrap_themes():
themelist = []
if not os.path.isdir(os.path.join(PROG_DIR, 'data', 'interfaces', 'bookstrap')):
return themelist # return empty if bookstrap interface not installed
if not internet():
logger.warn('Build Bookstrap Themes: No internet connection')
return themelist
URL = 'http://bootswatch.com/api/3.json'
result, success = fetchURL(URL, None, False) # use default headers, no retry
if not success:
logger.debug("Error getting bookstrap themes : %s" % result)
return themelist
try:
results = json.loads(result)
for theme in results['themes']:
themelist.append(theme['name'].lower())
except Exception as e:
# error reading results
logger.debug('JSON Error reading bookstrap themes, %s' % str(e))
logger.debug("Bookstrap found %i themes" % len(themelist))
return themelist
def checkForUpdates():
""" Called from webserver with thread name WEBSERVER, or as a cron job """
if 'Thread-' in threading.currentThread().name:
threading.currentThread().name = "CRON-VERSIONCHECK"
logger.debug(
'Set Install Type, Current & Latest Version and Commit status')
if internet():
getInstallType()
lazylibrarian.CONFIG['CURRENT_VERSION'] = getCurrentVersion()
lazylibrarian.CONFIG['LATEST_VERSION'] = getLatestVersion()
lazylibrarian.CONFIG[
'COMMITS_BEHIND'], lazylibrarian.COMMIT_LIST = getCommitDifferenceFromGit(
)
logger.debug('Update check complete')
else:
logger.warn('No internet connection')
def search_rss_book(books=None, reset=False):
try:
threadname = threading.currentThread().name
if "Thread-" in threadname:
if books is None:
threading.currentThread().name = "SEARCHALLRSS"
else:
threading.currentThread().name = "SEARCHRSS"
if not (lazylibrarian.USE_RSS()):
logger.warn('RSS search is disabled')
scheduleJob(action='Stop', target='search_rss_book')
return
if not internet():
logger.warn('Search RSS Book: No internet connection')
return
myDB = database.DBConnection()
resultlist, wishproviders = IterateOverGoodReads()
if not wishproviders:
logger.debug('No rss wishlists are set')
else:
# for each item in resultlist, add to database if necessary, and mark as wanted
for book in resultlist:
# we get rss_author, rss_title, rss_isbn, rss_bookid (goodreads bookid)
# we can just use bookid if goodreads, or try isbn and name matching on author/title if googlebooks
# not sure if anyone would use a goodreads wishlist if not using goodreads interface...
logger.debug('Processing %s item%s in wishlists' %
(len(resultlist), plural(len(resultlist))))
if book['rss_bookid'] and lazylibrarian.CONFIG[
'BOOK_API'] == "GoodReads":
bookmatch = myDB.match(
'select Status,BookName from books where bookid="%s"' %
book['rss_bookid'])
if bookmatch:
bookstatus = bookmatch['Status']
bookname = bookmatch['BookName']
if bookstatus in ['Open', 'Wanted', 'Have']:
logger.info(
u'Found book %s, already marked as "%s"' %
(bookname, bookstatus))
else: # skipped/ignored
logger.info(u'Found book %s, marking as "Wanted"' %
bookname)
controlValueDict = {"BookID": bookid}
newValueDict = {"Status": "Wanted"}
myDB.upsert("books", newValueDict,
controlValueDict)
else:
import_book(book['rss_bookid'])
else:
item = {}
headers = []
item['Title'] = book['rss_title']
if book['rss_bookid']:
item['BookID'] = book['rss_bookid']
headers.append('BookID')
if book['rss_isbn']:
item['ISBN'] = book['rss_isbn']
headers.append('ISBN')
bookmatch = finditem(item, book['rss_author'], headers)
if bookmatch: # it's already in the database
authorname = bookmatch['AuthorName']
bookname = bookmatch['BookName']
bookid = bookmatch['BookID']
bookstatus = bookmatch['Status']
if bookstatus in ['Open', 'Wanted', 'Have']:
logger.info(
u'Found book %s by %s, already marked as "%s"'
% (bookname, authorname, bookstatus))
else: # skipped/ignored
logger.info(
u'Found book %s by %s, marking as "Wanted"' %
(bookname, authorname))
controlValueDict = {"BookID": bookid}
newValueDict = {"Status": "Wanted"}
myDB.upsert("books", newValueDict,
controlValueDict)
else: # not in database yet
results = ''
if book['rss_isbn']:
results = search_for(book['rss_isbn'])
if results:
result = results[0]
if result['isbn_fuzz'] > lazylibrarian.CONFIG[
'MATCH_RATIO']:
logger.info(
"Found (%s%%) %s: %s" %
(result['isbn_fuzz'], result['authorname'],
result['bookname']))
import_book(result['bookid'])
bookmatch = True
if not results:
searchterm = "%s <ll> %s" % (
item['Title'],
formatAuthorName(book['rss_author']))
results = search_for(unaccented(searchterm))
if results:
result = results[0]
if result['author_fuzz'] > lazylibrarian.CONFIG['MATCH_RATIO'] \
and result['book_fuzz'] > lazylibrarian.CONFIG['MATCH_RATIO']:
logger.info(
"Found (%s%% %s%%) %s: %s" %
(result['author_fuzz'],
result['book_fuzz'], result['authorname'],
result['bookname']))
import_book(result['bookid'])
bookmatch = True
if not bookmatch:
msg = "Skipping book %s by %s" % (item['Title'],
book['rss_author'])
# noinspection PyUnboundLocalVariable
if not results:
msg += ', No results returned'
logger.warn(msg)
else:
msg += ', No match found'
logger.warn(msg)
msg = "Closest match (%s%% %s%%) %s: %s" % (
result['author_fuzz'], result['book_fuzz'],
result['authorname'], result['bookname'])
logger.warn(msg)
if books is None:
# We are performing a backlog search
cmd = 'SELECT BookID, AuthorName, Bookname, BookSub, BookAdded from books,authors '
cmd += 'WHERE books.AuthorID = authors.AuthorID and books.Status="Wanted" order by BookAdded desc'
searchbooks = myDB.select(cmd)
else:
# The user has added a new book
searchbooks = []
for book in books:
cmd = 'SELECT BookID, AuthorName, BookName, BookSub from books,authors '
cmd += 'WHERE books.AuthorID = authors.AuthorID and BookID="%s" ' % book[
'bookid']
cmd += 'AND books.Status="Wanted"'
searchbook = myDB.select(cmd)
for terms in searchbook:
searchbooks.append(terms)
if len(searchbooks) == 0:
return
resultlist, nproviders = IterateOverRSSSites()
if not nproviders:
if not wishproviders:
logger.warn('No rss providers are set, check config')
return # No point in continuing
logger.info('RSS Searching for %i book%s' %
(len(searchbooks), plural(len(searchbooks))))
rss_count = 0
for book in searchbooks:
authorname, bookname = get_searchterm(book, "book")
found = processResultList(resultlist, authorname, bookname, book,
'book')
# if you can't find the book, try title without any "(extended details, series etc)"
if not found and '(' in bookname: # anything to shorten?
authorname, bookname = get_searchterm(book, "shortbook")
found = processResultList(resultlist, authorname, bookname,
book, 'shortbook')
if not found:
logger.debug(
"Searches returned no results. Adding book %s - %s to queue."
% (authorname, bookname))
if found > True:
rss_count += 1
logger.info("RSS Search for Wanted items complete, found %s book%s" %
(rss_count, plural(rss_count)))
if reset:
scheduleJob(action='Restart', target='search_rss_book')
except Exception:
logger.error('Unhandled exception in search_rss_book: %s' %
traceback.format_exc())
def searchItem(item=None, bookid=None):
"""
Call all active search providers asking for a "general" search for item
return a list of results, each entry in list containing percentage_match, title, provider, size, url
"""
results = []
if not item:
return results
if not internet():
logger.debug('Search Item: No internet connection')
return results
book = {}
searchterm = unaccented_str(item)
book['searchterm'] = searchterm
if bookid:
book['bookid'] = bookid
else:
book['bookid'] = searchterm
nproviders = lazylibrarian.USE_NZB() + lazylibrarian.USE_TOR(
) + lazylibrarian.USE_RSS()
logger.debug('Searching %s providers for %s' % (nproviders, searchterm))
if lazylibrarian.USE_NZB():
resultlist, nproviders = IterateOverNewzNabSites(book, 'general')
if nproviders:
results += resultlist
if lazylibrarian.USE_TOR():
resultlist, nproviders = IterateOverTorrentSites(book, 'general')
if nproviders:
results += resultlist
if lazylibrarian.USE_RSS():
resultlist, nproviders = IterateOverRSSSites()
if nproviders:
results += resultlist
# reprocess to get consistent results
searchresults = []
for item in results:
provider = ''
title = ''
url = ''
size = ''
date = ''
mode = ''
if 'nzbtitle' in item:
title = item['nzbtitle']
if 'nzburl' in item:
url = item['nzburl']
if 'nzbprov' in item:
provider = item['nzbprov']
if 'nzbsize' in item:
size = item['nzbsize']
if 'nzbdate' in item:
date = item['nzbdate']
if 'nzbmode' in item:
mode = item['nzbmode']
if 'tor_title' in item:
title = item['tor_title']
if 'tor_url' in item:
url = item['tor_url']
if 'tor_prov' in item:
provider = item['tor_prov']
if 'tor_size' in item:
size = item['tor_size']
if 'tor_date' in item:
date = item['tor_date']
if 'tor_type' in item:
mode = item['tor_type']
if title and provider and mode and url:
# Not all results have a date or a size
if not date:
date = 'Fri, 01 Jan 1970 00:00:00 +0100'
if not size:
size = '1000'
# calculate match percentage
score = fuzz.token_set_ratio(searchterm, title)
# lose a point for each extra word in the title so we get the closest match
words = len(getList(searchterm))
words -= len(getList(title))
score -= abs(words)
if score >= 40: # ignore wildly wrong results?
url = url.split('?')[0]
result = {
'score': score,
'title': title,
'provider': provider,
'size': size,
'date': date,
'url': urllib.quote_plus(url),
'mode': mode
}
searchresults.append(result)
# from operator import itemgetter
# searchresults = sorted(searchresults, key=itemgetter('score'), reverse=True)
logger.debug('Found %s results for %s' % (len(searchresults), searchterm))
return searchresults
def search_magazines(mags=None, reset=False):
# produce a list of magazines to search for, tor, nzb, torznab, rss
try:
threadname = threading.currentThread().name
if "Thread-" in threadname:
if mags is None:
threading.currentThread().name = "SEARCHALLMAG"
else:
threading.currentThread().name = "SEARCHMAG"
if not internet():
logger.warn('Search Magazines: No internet connection')
return
myDB = database.DBConnection()
searchlist = []
if mags is None: # backlog search
searchmags = myDB.select('SELECT Title, Regex, LastAcquired, \
IssueDate from magazines WHERE Status="Active"'
)
else:
searchmags = []
for magazine in mags:
searchmags_temp = myDB.select(
'SELECT Title, Regex, LastAcquired, IssueDate from magazines \
WHERE Title="%s" AND Status="Active"'
% (magazine['bookid']))
for terms in searchmags_temp:
searchmags.append(terms)
if len(searchmags) == 0:
return
# should clear old search results as might not be available any more
# ie torrent not available, changed providers, out of news server retention etc.
# Only delete the "skipped" ones, not wanted/snatched/processed/ignored
logger.debug(u"Removing old magazine search results")
myDB.action('DELETE from pastissues WHERE Status="Skipped"')
logger.info('Searching for %i magazine%s' %
(len(searchmags), plural(len(searchmags))))
for searchmag in searchmags:
bookid = searchmag['Title']
searchterm = searchmag['Regex']
if not searchterm:
searchterm = searchmag['Title']
dic = {
'...': '',
' & ': ' ',
' = ': ' ',
'?': '',
'$': 's',
' + ': ' ',
'"': '',
',': '',
'*': ''
}
searchterm = unaccented_str(replace_all(searchterm, dic))
searchterm = re.sub('[.\-/]', ' ', searchterm).encode(
lazylibrarian.SYS_ENCODING)
searchlist.append({"bookid": bookid, "searchterm": searchterm})
if not searchlist:
logger.warn(
'There is nothing to search for. Mark some magazines as active.'
)
for book in searchlist:
resultlist = []
if lazylibrarian.USE_NZB():
resultlist, nproviders = IterateOverNewzNabSites(book, 'mag')
if not nproviders:
logger.warn(
'No nzb providers are set. Check config for NEWZNAB or TORZNAB providers'
)
if lazylibrarian.USE_TOR():
tor_resultlist, nproviders = IterateOverTorrentSites(
book, 'mag')
if not nproviders:
logger.warn(
'No torrent providers are set. Check config for TORRENT providers'
)
if tor_resultlist:
for item in tor_resultlist: # reformat the torrent results so they look like nzbs
resultlist.append({
'bookid': item['bookid'],
'nzbprov': item['tor_prov'],
'nzbtitle': item['tor_title'],
'nzburl': item['tor_url'],
'nzbdate':
'Fri, 01 Jan 1970 00:00:00 +0100', # fake date as none returned from torrents
'nzbsize': item['tor_size'],
'nzbmode': 'torrent'
})
if lazylibrarian.USE_RSS():
rss_resultlist, nproviders = IterateOverRSSSites()
if not nproviders:
logger.warn(
'No rss providers are set. Check config for RSS providers'
)
if rss_resultlist:
for item in rss_resultlist: # reformat the rss results so they look like nzbs
resultlist.append({
'bookid': book['bookid'],
'nzbprov': item['tor_prov'],
'nzbtitle': item['tor_title'],
'nzburl': item['tor_url'],
'nzbdate': item[
'tor_date'], # may be fake date as none returned from rss torrents, only rss nzb
'nzbsize': item['tor_size'],
'nzbmode': item['tor_type']
})
if not resultlist:
logger.debug("Adding magazine %s to queue." %
book['searchterm'])
else:
bad_name = 0
bad_date = 0
old_date = 0
rejects = 0
total_nzbs = 0
new_date = 0
maglist = []
issues = []
bookid = ''
for nzb in resultlist:
total_nzbs += 1
bookid = nzb['bookid']
nzbtitle = unaccented_str(nzb['nzbtitle'])
nzbtitle = nzbtitle.replace('"', '').replace(
"'", "") # suppress " in titles
nzburl = nzb['nzburl']
nzbprov = nzb['nzbprov']
nzbdate_temp = nzb['nzbdate']
nzbsize_temp = nzb['nzbsize']
nzbsize_temp = check_int(
nzbsize_temp, 1000
) # not all torrents returned by torznab have a size
nzbsize = round(float(nzbsize_temp) / 1048576, 2)
nzbdate = nzbdate2format(nzbdate_temp)
nzbmode = nzb['nzbmode']
results = myDB.match(
'SELECT * from magazines WHERE Title="%s"' % bookid)
if not results:
logger.debug(
'Magazine [%s] does not match search term [%s].' %
(nzbtitle, bookid))
bad_name += 1
else:
rejected = False
maxsize = check_int(
lazylibrarian.CONFIG['REJECT_MAGSIZE'], 0)
if maxsize and nzbsize > maxsize:
logger.debug("Rejecting %s, too large" % nzbtitle)
rejected = True
if not rejected:
minsize = check_int(
lazylibrarian.CONFIG['REJECT_MAGMIN'], 0)
if minsize and nzbsize < minsize:
logger.debug("Rejecting %s, too small" %
nzbtitle)
rejected = True
if not rejected:
control_date = results['IssueDate']
dic = {
'.': ' ',
'-': ' ',
'/': ' ',
'+': ' ',
'_': ' ',
'(': '',
')': ''
}
nzbtitle_formatted = replace_all(nzbtitle,
dic).strip()
# Need to make sure that substrings of magazine titles don't get found
# (e.g. Maxim USA will find Maximum PC USA) - token_set_ratio takes care of this
# remove extra spaces if they're in a row
if nzbtitle_formatted[
0] == '[' and nzbtitle_formatted[-1] == ']':
nzbtitle_formatted = nzbtitle_formatted[1:-1]
nzbtitle_exploded_temp = " ".join(
nzbtitle_formatted.split())
nzbtitle_exploded = nzbtitle_exploded_temp.split(
' ')
if ' ' in bookid:
bookid_exploded = bookid.split(' ')
else:
bookid_exploded = [bookid]
# check nzb has magazine title and a date/issue nr
# eg The MagPI July 2015
if len(nzbtitle_exploded) > len(bookid_exploded):
# needs to be longer as it has to include a date
# check (nearly) all the words in the mag title are in the nzbtitle - allow some fuzz
mag_title_match = fuzz.token_set_ratio(
unaccented(bookid),
unaccented(nzbtitle_formatted))
if mag_title_match < lazylibrarian.CONFIG[
'MATCH_RATIO']:
logger.debug(
u"Magazine token set Match failed: " +
str(mag_title_match) + "% for " +
nzbtitle_formatted)
rejected = True
else:
logger.debug(u"Magazine matched: " +
str(mag_title_match) + "% " +
bookid + " for " +
nzbtitle_formatted)
else:
logger.debug("Magazine name too short (%s)" %
len(nzbtitle_exploded))
rejected = True
if not rejected:
already_failed = myDB.match(
'SELECT * from wanted WHERE NZBurl="%s" and Status="Failed"'
% nzburl)
if already_failed:
logger.debug(
"Rejecting %s, blacklisted at %s" %
(nzbtitle_formatted,
already_failed['NZBprov']))
rejected = True
if not rejected:
reject_list = getList(results['Reject'])
lower_title = unaccented(
nzbtitle_formatted).lower()
lower_bookid = unaccented(bookid).lower()
for word in reject_list:
if word in lower_title and word not in lower_bookid:
rejected = True
logger.debug("Rejecting %s, contains %s" %
(nzbtitle_formatted, word))
break
regex_pass = 0
if not rejected:
# Magazine names have many different styles of date
# DD MonthName YYYY OR MonthName YYYY or Issue nn, MonthName YYYY
# MonthName DD YYYY or MonthName DD, YYYY
# YYYY MM or YYYY MM DD
# Issue/No/Nr/Vol nn, YYYY or Issue/No/Nr/Vol nn
# nn YYYY issue number without "Nr" before it
# issue and year as a single 6 digit string eg 222015
newdatish = "none"
# DD MonthName YYYY OR MonthName YYYY or Issue nn, MonthName YYYY
pos = 0
while pos < len(nzbtitle_exploded):
year = check_year(nzbtitle_exploded[pos])
if year and pos:
month = month2num(nzbtitle_exploded[pos -
1])
if month:
if pos - 1:
day = check_int(
nzbtitle_exploded[pos - 2], 1)
if day > 31: # probably issue number nn
day = 1
else:
day = 1
newdatish = "%04d-%02d-%02d" % (
year, month, day)
try:
_ = datetime.date(year, month, day)
regex_pass = 1
break
except ValueError:
regex_pass = 0
pos += 1
# MonthName DD YYYY or MonthName DD, YYYY
if not regex_pass:
pos = 0
while pos < len(nzbtitle_exploded):
year = check_year(nzbtitle_exploded[pos])
if year and (pos - 1):
month = month2num(
nzbtitle_exploded[pos - 2])
if month:
day = check_int(
nzbtitle_exploded[
pos - 1].rstrip(','), 1)
try:
_ = datetime.date(
year, month, day)
newdatish = "%04d-%02d-%02d" % (
year, month, day)
regex_pass = 2
break
except ValueError:
regex_pass = 0
pos += 1
# YYYY MM or YYYY MM DD
if not regex_pass:
pos = 0
while pos < len(nzbtitle_exploded):
year = check_year(nzbtitle_exploded[pos])
if year and pos + 1 < len(
nzbtitle_exploded):
month = check_int(
nzbtitle_exploded[pos + 1], 0)
if month:
if pos + 2 < len(
nzbtitle_exploded):
day = check_int(
nzbtitle_exploded[pos + 2],
1)
else:
day = 1
try:
_ = datetime.date(
year, month, day)
newdatish = "%04d-%02d-%02d" % (
year, month, day)
regex_pass = 3
break
except ValueError:
regex_pass = 0
pos += 1
# Issue/No/Nr/Vol nn, YYYY or Issue/No/Nr/Vol nn
if not regex_pass:
pos = 0
while pos < len(nzbtitle_exploded):
if nzbtitle_exploded[pos].lower() in [
"issue", "no", "nr", "vol"
]:
if pos + 1 < len(nzbtitle_exploded):
issue = check_int(
nzbtitle_exploded[pos + 1], 0)
if issue:
newdatish = str(
issue) # 4 == 04 == 004
if pos + 2 < len(
nzbtitle_exploded):
year = check_year(
nzbtitle_exploded[pos +
2])
if year and year < int(
datetime.date.
today().year):
newdatish = '0' # it's old
regex_pass = 4 # Issue/No/Nr/Vol nn, YYYY
else:
regex_pass = 5 # Issue/No/Nr/Vol nn
break
pos += 1
# nn YYYY issue number without "Nr" before it
if not regex_pass:
pos = 1
while pos < len(nzbtitle_exploded):
year = check_year(nzbtitle_exploded[pos])
if year:
issue = check_int(
nzbtitle_exploded[pos - 1], 0)
if issue:
newdatish = str(
issue) # 4 == 04 == 004
regex_pass = 6
if year < int(datetime.date.today(
).year):
newdatish = '0' # it's old
break
pos += 1
# issue and year as a single 6 digit string eg 222015
if not regex_pass:
pos = 0
while pos < len(nzbtitle_exploded):
issue = nzbtitle_exploded[pos]
if issue.isdigit() and len(issue) == 6:
year = int(issue[2:])
issue = int(issue[:2])
newdatish = str(
issue) # 4 == 04 == 004
regex_pass = 7
if year < int(
datetime.date.today().year):
newdatish = '0' # it's old
break
pos += 1
if not regex_pass:
logger.debug(
'Magazine %s not in a recognised date format.'
% nzbtitle_formatted)
bad_date += 1
# allow issues with good name but bad date to be included
# so user can manually select them, incl those with issue numbers
newdatish = "1970-01-01" # provide a fake date for bad-date issues
regex_pass = 99
if rejected:
rejects += 1
else:
if lazylibrarian.LOGLEVEL > 3:
logger.debug("regex %s [%s] %s" %
(regex_pass, nzbtitle_formatted,
newdatish))
# wanted issues go into wanted table marked "Wanted"
# the rest into pastissues table marked "Skipped"
insert_table = "pastissues"
insert_status = "Skipped"
if control_date is None: # we haven't got any copies of this magazine yet
# get a rough time just over a month ago to compare to, in format yyyy-mm-dd
# could perhaps calc differently for weekly, biweekly etc
# or for magazines with only an issue number, use zero
if '-' in str(newdatish):
start_time = time.time()
start_time -= int(
lazylibrarian.CONFIG['MAG_AGE']
) * 24 * 60 * 60 # number of seconds in days
if start_time < 0: # limit of unixtime (1st Jan 1970)
start_time = 0
control_date = time.strftime(
"%Y-%m-%d", time.localtime(start_time))
logger.debug(
'Magazine date comparing to %s' %
control_date)
else:
control_date = 0
if '-' in str(control_date) and '-' in str(
newdatish):
# only grab a copy if it's newer than the most recent we have,
# or newer than a month ago if we have none
comp_date = datecompare(
newdatish, control_date)
elif '-' not in str(
control_date) and '-' not in str(
newdatish):
# for issue numbers, check if later than last one we have
comp_date = int(newdatish) - int(control_date)
newdatish = "%s" % newdatish
newdatish = newdatish.zfill(
4) # pad so we sort correctly
else:
# invalid comparison of date and issue number
logger.debug(
'Magazine %s incorrect date or issue format.'
% nzbtitle_formatted)
bad_date += 1
newdatish = "1970-01-01" # this is our fake date for ones we can't decipher
comp_date = 0
if comp_date > 0:
# keep track of what we're going to download so we don't download dupes
new_date += 1
issue = bookid + ',' + newdatish
if issue not in issues:
maglist.append({
'bookid': bookid,
'nzbprov': nzbprov,
'nzbtitle': nzbtitle,
'nzburl': nzburl,
'nzbmode': nzbmode
})
logger.debug(
'This issue of %s is new, downloading'
% nzbtitle_formatted)
issues.append(issue)
insert_table = "wanted"
insert_status = "Wanted"
nzbdate = now() # when we asked for it
else:
logger.debug(
'This issue of %s is already flagged for download'
% issue)
else:
if newdatish != "1970-01-01": # this is our fake date for ones we can't decipher
logger.debug(
'This issue of %s is old; skipping.' %
nzbtitle_formatted)
old_date += 1
# store only the _new_ matching results
# Don't add a new entry if this issue has been found on an earlier search
# and status has been user-set ( we only delete the "Skipped" ones )
# In "wanted" table it might be already snatched/downloading/processing
mag_entry = myDB.match(
'SELECT * from %s WHERE NZBtitle="%s" and NZBprov="%s"'
% (insert_table, nzbtitle, nzbprov))
if not mag_entry:
controlValueDict = {
"NZBtitle": nzbtitle,
"NZBprov": nzbprov
}
newValueDict = {
"NZBurl": nzburl,
"BookID": bookid,
"NZBdate": nzbdate,
"AuxInfo": newdatish,
"Status": insert_status,
"NZBsize": nzbsize,
"NZBmode": nzbmode
}
myDB.upsert(insert_table, newValueDict,
controlValueDict)
logger.info(
'Found %i result%s for %s. %i new, %i old, %i fail date, %i fail name, %i rejected: %i to download'
% (total_nzbs, plural(total_nzbs), bookid, new_date,
old_date, bad_date, bad_name, rejects, len(maglist)))
for magazine in maglist:
if magazine['nzbmode'] in ["torznab", "torrent", "magnet"]:
snatch = TORDownloadMethod(magazine['bookid'],
magazine['nzbtitle'],
magazine['nzburl'])
else:
snatch = NZBDownloadMethod(magazine['bookid'],
magazine['nzbtitle'],
magazine['nzburl'])
if snatch:
logger.info(
'Downloading %s from %s' %
(magazine['nzbtitle'], magazine["nzbprov"]))
notify_snatch("%s from %s at %s" % (unaccented(
magazine['nzbtitle']), magazine["nzbprov"], now()))
custom_notify_snatch(magazine['bookid'])
scheduleJob(action='Start', target='processDir')
if reset:
scheduleJob(action='Restart', target='search_magazines')
logger.info("Search for magazines complete")
except Exception:
logger.error('Unhandled exception in search_magazines: %s' %
traceback.format_exc())
def search_tor_book(books=None, reset=False):
try:
threadname = threading.currentThread().name
if "Thread-" in threadname:
if books is None:
threading.currentThread().name = "SEARCHALLTOR"
else:
threading.currentThread().name = "SEARCHTOR"
if not lazylibrarian.USE_TOR():
logger.warn('No Torrent providers set, check config')
return
if not internet():
logger.warn('Search Tor Book: No internet connection')
return
myDB = database.DBConnection()
searchlist = []
if books is None:
# We are performing a backlog search
cmd = 'SELECT BookID, AuthorName, Bookname, BookSub, BookAdded from books,authors '
cmd += 'WHERE books.AuthorID = authors.AuthorID and books.Status="Wanted" order by BookAdded desc'
searchbooks = myDB.select(cmd)
else:
# The user has added a new book
searchbooks = []
for book in books:
cmd = 'SELECT BookID, AuthorName, BookName, BookSub from books,authors WHERE books.Status="Wanted"'
cmd += ' and books.AuthorID = authors.AuthorID and BookID="%s"' % book['bookid']
searchbook = myDB.select(cmd)
for terms in searchbook:
searchbooks.append(terms)
if len(searchbooks) == 0:
return
logger.info('TOR Searching for %i book%s' % (len(searchbooks), plural(len(searchbooks))))
for searchbook in searchbooks:
# searchterm is only used for display purposes
searchterm = searchbook['AuthorName'] + ' ' + searchbook['BookName']
if searchbook['BookSub']:
searchterm = searchterm + ': ' + searchbook['BookSub']
searchlist.append(
{"bookid": searchbook['BookID'],
"bookName": searchbook['BookName'],
"bookSub": searchbook['BookSub'],
"authorName": searchbook['AuthorName'],
"searchterm": searchterm})
tor_count = 0
for book in searchlist:
resultlist, nproviders = IterateOverTorrentSites(book, 'book')
if not nproviders:
logger.warn('No torrent providers are set, check config')
return # No point in continuing
found = processResultList(resultlist, book, "book")
# if you can't find the book, try author/title without any "(extended details, series etc)"
if not found and '(' in book['bookName']:
resultlist, nproviders = IterateOverTorrentSites(book, 'shortbook')
found = processResultList(resultlist, book, "shortbook")
# general search is the same as booksearch for torrents
# if not found:
# resultlist, nproviders = IterateOverTorrentSites(book, 'general')
# found = processResultList(resultlist, book, "general")
if not found:
logger.debug("Searches for %s returned no results." % book['searchterm'])
if found > True:
tor_count += 1
logger.info("TORSearch for Wanted items complete, found %s book%s" % (tor_count, plural(tor_count)))
if reset:
scheduleJob(action='Restart', target='search_tor_book')
except Exception:
logger.error('Unhandled exception in search_tor_book: %s' % traceback.format_exc())
def search_nzb_book(books=None, reset=False):
try:
threadname = threading.currentThread().name
if "Thread-" in threadname:
if books is None:
threading.currentThread().name = "SEARCHALLNZB"
else:
threading.currentThread().name = "SEARCHNZB"
if not lazylibrarian.USE_NZB():
logger.warn('No NEWZNAB/TORZNAB providers set, check config')
return
if not internet():
logger.warn('Search NZB Book: No internet connection')
return
myDB = database.DBConnection()
searchlist = []
if books is None:
# We are performing a backlog search
cmd = 'SELECT BookID, AuthorName, Bookname, BookSub, BookAdded from books,authors '
cmd += 'WHERE books.Status="Wanted" and books.AuthorID = authors.AuthorID order by BookAdded desc'
searchbooks = myDB.select(cmd)
else:
# The user has added a new book
searchbooks = []
for book in books:
cmd = 'SELECT BookID, AuthorName, BookName, BookSub from books,authors'
cmd += ' WHERE BookID="%s"' % book['bookid']
cmd += ' AND books.AuthorID = authors.AuthorID AND books.Status="Wanted"'
searchbook = myDB.select(cmd)
for terms in searchbook:
searchbooks.append(terms)
if len(searchbooks) == 0:
return
logger.info('NZB Searching for %i book%s' %
(len(searchbooks), plural(len(searchbooks))))
for searchbook in searchbooks:
# searchterm is only used for display purposes
searchterm = searchbook['AuthorName'] + ' ' + searchbook['BookName']
if searchbook['BookSub']:
searchterm = searchterm + ': ' + searchbook['BookSub']
searchlist.append({
"bookid": searchbook['BookID'],
"bookName": searchbook['BookName'],
"bookSub": searchbook['BookSub'],
"authorName": searchbook['AuthorName'],
"searchterm": searchterm
})
nzb_count = 0
for book in searchlist:
# first attempt, try author/title in category "book"
resultlist, nproviders = providers.IterateOverNewzNabSites(
book, 'book')
if not nproviders:
logger.warn(
'No NewzNab or TorzNab providers are set, check config')
return # no point in continuing
found = processResultList(resultlist, book, "book")
# if you can't find the book, try author/title without any "(extended details, series etc)"
if not found and '(' in book['bookName']:
resultlist, nproviders = providers.IterateOverNewzNabSites(
book, 'shortbook')
found = processResultList(resultlist, book, "shortbook")
# if you can't find the book under "books", you might find under general search
if not found:
resultlist, nproviders = providers.IterateOverNewzNabSites(
book, 'general')
found = processResultList(resultlist, book, "general")
# if still not found, try general search again without any "(extended details, series etc)"
if not found and '(' in book['bookName']:
resultlist, nproviders = providers.IterateOverNewzNabSites(
book, 'shortgeneral')
found = processResultList(resultlist, book, "shortgeneral")
if not found:
logger.info("NZB Searches for %s returned no results." %
book['searchterm'])
if found > True:
nzb_count += 1 # we found it
logger.info("NZBSearch for Wanted items complete, found %s book%s" %
(nzb_count, plural(nzb_count)))
if reset:
scheduleJob(action='Restart', target='search_nzb_book')
except Exception:
logger.error('Unhandled exception in search_nzb_book: %s' %
traceback.format_exc())
def LibraryScan(startdir=None):
""" Scan a directory tree adding new books into database
Return how many books you added """
try:
destdir = lazylibrarian.DIRECTORY('Destination')
if not startdir:
if not destdir:
logger.warn('Cannot find destination directory: %s. Not scanning' % destdir)
return 0
startdir = destdir
if not os.path.isdir(startdir):
logger.warn('Cannot find directory: %s. Not scanning' % startdir)
return 0
if not internet():
logger.warn('Libraryscan: No internet connection')
return 0
myDB = database.DBConnection()
# keep statistics of full library scans
if startdir == destdir:
myDB.action('DELETE from stats')
try: # remove any extra whitespace in authornames
authors = myDB.select('SELECT AuthorID,AuthorName FROM authors WHERE AuthorName like "% %"')
if authors:
logger.info('Removing extra spaces from %s authorname%s' % (len(authors), plural(len(authors))))
for author in authors:
authorid = author["AuthorID"]
authorname = ' '.join(author['AuthorName'].split())
# Have we got author name both with-and-without extra spaces? If so, merge them
duplicate = myDB.match(
'Select AuthorID,AuthorName FROM authors WHERE AuthorName="%s"' % authorname)
if duplicate:
myDB.action('DELETE from authors where authorname="%s"' % author['AuthorName'])
if author['AuthorID'] != duplicate['AuthorID']:
myDB.action('UPDATE books set AuthorID="%s" WHERE AuthorID="%s"' %
(duplicate['AuthorID'], author['AuthorID']))
else:
myDB.action(
'UPDATE authors set AuthorName="%s" WHERE AuthorID="%s"' % (authorname, authorid))
except Exception as e:
logger.info('Error: ' + str(e))
logger.info('Scanning ebook directory: %s' % startdir)
new_book_count = 0
modified_count = 0
rescan_count = 0
rescan_hits = 0
file_count = 0
author = ""
if lazylibrarian.CONFIG['FULL_SCAN']:
cmd = 'select AuthorName, BookName, BookFile, BookID from books,authors'
cmd += ' where books.AuthorID = authors.AuthorID and books.Status="Open"'
if not startdir == destdir:
cmd += ' and BookFile like "' + startdir + '%"'
books = myDB.select(cmd)
status = lazylibrarian.CONFIG['NOTFOUND_STATUS']
logger.info('Missing books will be marked as %s' % status)
for book in books:
bookID = book['BookID']
bookfile = book['BookFile']
if not (bookfile and os.path.isfile(bookfile)):
myDB.action('update books set Status="%s" where BookID="%s"' % (status, bookID))
myDB.action('update books set BookFile="" where BookID="%s"' % bookID)
logger.warn('Book %s - %s updated as not found on disk' % (book['AuthorName'], book['BookName']))
# to save repeat-scans of the same directory if it contains multiple formats of the same book,
# keep track of which directories we've already looked at
processed_subdirectories = []
warned = False # have we warned about no new authors setting
matchString = ''
for char in lazylibrarian.CONFIG['EBOOK_DEST_FILE']:
matchString = matchString + '\\' + char
# massage the EBOOK_DEST_FILE config parameter into something we can use
# with regular expression matching
booktypes = ''
count = -1
booktype_list = getList(lazylibrarian.CONFIG['EBOOK_TYPE'])
for book_type in booktype_list:
count += 1
if count == 0:
booktypes = book_type
else:
booktypes = booktypes + '|' + book_type
matchString = matchString.replace("\\$\\A\\u\\t\\h\\o\\r", "(?P<author>.*?)").replace(
"\\$\\T\\i\\t\\l\\e", "(?P<book>.*?)") + '\.[' + booktypes + ']'
pattern = re.compile(matchString, re.VERBOSE)
for r, d, f in os.walk(startdir):
for directory in d[:]:
# prevent magazine being scanned
if directory.startswith("_") or directory.startswith("."):
d.remove(directory)
for files in f:
file_count += 1
if isinstance(r, str):
r = r.decode(lazylibrarian.SYS_ENCODING)
subdirectory = r.replace(startdir, '')
# Added new code to skip if we've done this directory before.
# Made this conditional with a switch in config.ini
# in case user keeps multiple different books in the same subdirectory
if lazylibrarian.CONFIG['IMP_SINGLEBOOK'] and (subdirectory in processed_subdirectories):
logger.debug("[%s] already scanned" % subdirectory)
else:
# If this is a book, try to get author/title/isbn/language
# if epub or mobi, read metadata from the book
# If metadata.opf exists, use that allowing it to override
# embedded metadata. User may have edited metadata.opf
# to merge author aliases together
# If all else fails, try pattern match for author/title
# and look up isbn/lang from LT or GR later
match = 0
if is_valid_booktype(files):
logger.debug("[%s] Now scanning subdirectory %s" % (startdir, subdirectory))
language = "Unknown"
isbn = ""
book = ""
author = ""
gr_id = ""
gb_id = ""
extn = os.path.splitext(files)[1]
# if it's an epub or a mobi we can try to read metadata from it
if (extn == ".epub") or (extn == ".mobi"):
book_filename = os.path.join(r, files).encode(lazylibrarian.SYS_ENCODING)
try:
res = get_book_info(book_filename)
except Exception as e:
logger.debug('get_book_info failed for %s, %s' % (book_filename, str(e)))
res = {}
# title and creator are the minimum we need
if 'title' in res and 'creator' in res:
book = res['title']
author = res['creator']
if book and len(book) > 2 and author and len(author) > 2:
match = 1
if 'language' in res:
language = res['language']
if 'identifier' in res:
isbn = res['identifier']
if 'type' in res:
extn = res['type']
logger.debug("book meta [%s] [%s] [%s] [%s] [%s]" %
(isbn, language, author, book, extn))
if not match:
logger.debug("Book meta incomplete in %s" % book_filename)
# calibre uses "metadata.opf", LL uses "bookname - authorname.opf"
# just look for any .opf file in the current directory since we don't know
# LL preferred authorname/bookname at this point.
# Allow metadata in file to override book contents as may be users pref
metafile = opf_file(r)
try:
res = get_book_info(metafile)
except Exception as e:
logger.debug('get_book_info failed for %s, %s' % (metafile, str(e)))
res = {}
# title and creator are the minimum we need
if 'title' in res and 'creator' in res:
book = res['title']
author = res['creator']
if book and len(book) > 2 and author and len(author) > 2:
match = 1
if 'language' in res:
language = res['language']
if 'identifier' in res:
isbn = res['identifier']
if 'gr_id' in res:
gr_id = res['gr_id']
logger.debug("file meta [%s] [%s] [%s] [%s] [%s]" % (isbn, language, author, book, gr_id))
if not match:
logger.debug("File meta incomplete in %s" % metafile)
if not match: # no author/book from metadata file, and not embedded either
match = pattern.match(files)
if match:
author = match.group("author")
book = match.group("book")
if len(book) <= 2 or len(author) <= 2:
match = 0
if not match:
logger.debug("Pattern match failed [%s]" % files)
if match:
# flag that we found a book in this subdirectory
processed_subdirectories.append(subdirectory)
# If we have a valid looking isbn, and language != "Unknown", add it to cache
if language != "Unknown" and is_valid_isbn(isbn):
logger.debug("Found Language [%s] ISBN [%s]" % (language, isbn))
# we need to add it to language cache if not already
# there, is_valid_isbn has checked length is 10 or 13
if len(isbn) == 10:
isbnhead = isbn[0:3]
else:
isbnhead = isbn[3:6]
match = myDB.match('SELECT lang FROM languages where isbn = "%s"' % isbnhead)
if not match:
myDB.action('insert into languages values ("%s", "%s")' % (isbnhead, language))
logger.debug("Cached Lang [%s] ISBN [%s]" % (language, isbnhead))
else:
logger.debug("Already cached Lang [%s] ISBN [%s]" % (language, isbnhead))
author, authorid, new = addAuthorNameToDB(author) # get the author name as we know it...
if author:
# author exists, check if this book by this author is in our database
# metadata might have quotes in book name
# some books might be stored under a different author name
# eg books by multiple authors, books where author is "writing as"
# or books we moved to "merge" authors
book = book.replace("'", "")
# First try and find it under author and bookname
# as we may have it under a different bookid or isbn to goodreads/googlebooks
# which might have several bookid/isbn for the same book
bookid = find_book_in_db(myDB, author, book)
if not bookid:
# Title or author name might not match or multiple authors
# See if the gr_id, gb_id is already in our database
if gr_id:
bookid = gr_id
elif gb_id:
bookid = gb_id
else:
bookid = ""
if bookid:
match = myDB.match('SELECT BookID FROM books where BookID = "%s"' % bookid)
if not match:
msg = 'Unable to find book %s by %s in database, trying to add it using '
if bookid == gr_id:
msg += "GoodReads ID " + gr_id
if bookid == gb_id:
msg += "GoogleBooks ID " + gb_id
logger.debug(msg % (book, author))
if lazylibrarian.CONFIG['BOOK_API'] == "GoodReads" and gr_id:
GR_ID = GoodReads(gr_id)
GR_ID.find_book(gr_id, None)
elif lazylibrarian.CONFIG['BOOK_API'] == "GoogleBooks" and gb_id:
GB_ID = GoogleBooks(gb_id)
GB_ID.find_book(gb_id, None)
# see if it's there now...
match = myDB.match('SELECT BookID from books where BookID="%s"' % bookid)
if not match:
logger.debug("Unable to add bookid %s to database" % bookid)
bookid = ""
if not bookid and isbn:
# See if the isbn is in our database
match = myDB.match('SELECT BookID FROM books where BookIsbn = "%s"' % isbn)
if match:
bookid = match['BookID']
if not bookid:
# get author name from parent directory of this book directory
newauthor = os.path.basename(os.path.dirname(r))
# calibre replaces trailing periods with _ eg Smith Jr. -> Smith Jr_
if newauthor.endswith('_'):
newauthor = newauthor[:-1] + '.'
if author.lower() != newauthor.lower():
logger.debug("Trying authorname [%s]" % newauthor)
bookid = find_book_in_db(myDB, newauthor, book)
if bookid:
logger.warn("%s not found under [%s], found under [%s]" %
(book, author, newauthor))
# at this point if we still have no bookid, it looks like we
# have author and book title but no database entry for it
if not bookid:
if lazylibrarian.CONFIG['BOOK_API'] == "GoodReads":
# Either goodreads doesn't have the book or it didn't match language prefs
# Since we have the book anyway, try and reload it ignoring language prefs
rescan_count += 1
base_url = 'http://www.goodreads.com/search.xml?q='
params = {"key": lazylibrarian.CONFIG['GR_API']}
if author[1] in '. ':
surname = author
forename = ''
while surname[1] in '. ':
forename = forename + surname[0] + '.'
surname = surname[2:].strip()
if author != forename + ' ' + surname:
logger.debug('Stripped authorname [%s] to [%s %s]' %
(author, forename, surname))
author = forename + ' ' + surname
author = ' '.join(author.split()) # ensure no extra whitespace
searchname = author + ' ' + book
searchname = cleanName(unaccented(searchname))
searchterm = urllib.quote_plus(searchname.encode(lazylibrarian.SYS_ENCODING))
set_url = base_url + searchterm + '&' + urllib.urlencode(params)
try:
rootxml, in_cache = get_xml_request(set_url)
if not len(rootxml):
logger.debug("Error requesting results from GoodReads")
else:
resultxml = rootxml.getiterator('work')
for item in resultxml:
booktitle = item.find('./best_book/title').text
book_fuzz = fuzz.token_set_ratio(booktitle, book)
if book_fuzz >= 98:
logger.debug("Rescan found %s : %s" % (booktitle, language))
rescan_hits += 1
bookid = item.find('./best_book/id').text
GR_ID = GoodReads(bookid)
GR_ID.find_book(bookid, None)
if language and language != "Unknown":
# set language from book metadata
logger.debug("Setting language from metadata %s : %s" % (booktitle, language))
myDB.action('UPDATE books SET BookLang="%s" WHERE BookID="%s"' %
(language, bookid))
break
if not bookid:
logger.warn("GoodReads doesn't know about %s" % book)
except Exception as e:
logger.error("Error finding rescan results: %s" % str(e))
elif lazylibrarian.CONFIG['BOOK_API'] == "GoogleBooks":
# if we get here using googlebooks it's because googlebooks
# doesn't have the book. No point in looking for it again.
logger.warn("GoogleBooks doesn't know about %s" % book)
# see if it's there now...
if bookid:
cmd = 'SELECT books.Status, BookFile, AuthorName, BookName from books,authors '
cmd += 'where books.AuthorID = authors.AuthorID and BookID="%s"' % bookid
check_status = myDB.match(cmd)
if not check_status:
logger.debug('Unable to find bookid %s in database' % bookid)
else:
if check_status['Status'] != 'Open':
# we found a new book
new_book_count += 1
myDB.action(
'UPDATE books set Status="Open" where BookID="%s"' % bookid)
# store book location so we can check if it gets removed
book_filename = os.path.join(r, files)
if not check_status['BookFile']: # no previous location
myDB.action('UPDATE books set BookFile="%s" where BookID="%s"' %
(book_filename, bookid))
# location may have changed since last scan
elif book_filename != check_status['BookFile']:
modified_count += 1
logger.warn("Updating book location for %s %s from %s to %s" %
(author, book, check_status['BookFile'], book_filename))
logger.debug("%s %s matched %s BookID %s, [%s][%s]" %
(author, book, check_status['Status'], bookid,
check_status['AuthorName'], check_status['BookName']))
myDB.action('UPDATE books set BookFile="%s" where BookID="%s"' %
(book_filename, bookid))
# update cover file to cover.jpg in book folder (if exists)
bookdir = os.path.dirname(book_filename)
coverimg = os.path.join(bookdir, 'cover.jpg')
if os.path.isfile(coverimg):
cachedir = lazylibrarian.CACHEDIR
cacheimg = os.path.join(cachedir, 'book', bookid + '.jpg')
copyfile(coverimg, cacheimg)
else:
logger.warn(
"Failed to match book [%s] by [%s] in database" % (book, author))
else:
if not warned and not lazylibrarian.CONFIG['ADD_AUTHOR']:
logger.warn("Add authors to database is disabled")
warned = True
logger.info("%s/%s new/modified book%s found and added to the database" %
(new_book_count, modified_count, plural(new_book_count + modified_count)))
logger.info("%s file%s processed" % (file_count, plural(file_count)))
if startdir == destdir:
# On full library scans, check for missing workpages
setWorkPages()
# and books with unknown language
nolang = myDB.match(
"select count('BookID') as counter from Books where status='Open' and BookLang='Unknown'")
nolang = nolang['counter']
if nolang:
logger.warn("Found %s book%s in your library with unknown language" % (nolang, plural(nolang)))
# show stats if new books were added
stats = myDB.match(
"SELECT sum(GR_book_hits), sum(GR_lang_hits), sum(LT_lang_hits), sum(GB_lang_change), \
sum(cache_hits), sum(bad_lang), sum(bad_char), sum(uncached), sum(duplicates) FROM stats")
st= {'GR_book_hits': stats['sum(GR_book_hits)'], 'GB_book_hits': stats['sum(GR_book_hits)'],
'GR_lang_hits': stats['sum(GR_lang_hits)'], 'LT_lang_hits': stats['sum(LT_lang_hits)'],
'GB_lang_change': stats['sum(GB_lang_change)'], 'cache_hits': stats['sum(cache_hits)'],
'bad_lang': stats['sum(bad_lang)'], 'bad_char': stats['sum(bad_char)'],
'uncached': stats['sum(uncached)'], 'duplicates': stats['sum(duplicates)']}
for item in st.keys():
if st[item] is None:
st[item] = 0
if lazylibrarian.CONFIG['BOOK_API'] == "GoogleBooks":
logger.debug("GoogleBooks was hit %s time%s for books" %
(st['GR_book_hits'], plural(st['GR_book_hits'])))
logger.debug("GoogleBooks language was changed %s time%s" %
(st['GB_lang_change'], plural(st['GB_lang_change'])))
if lazylibrarian.CONFIG['BOOK_API'] == "GoodReads":
logger.debug("GoodReads was hit %s time%s for books" %
(st['GR_book_hits'], plural(st['GR_book_hits'])))
logger.debug("GoodReads was hit %s time%s for languages" %
(st['GR_lang_hits'], plural(st['GR_lang_hits'])))
logger.debug("LibraryThing was hit %s time%s for languages" %
(st['LT_lang_hits'], plural(st['LT_lang_hits'])))
logger.debug("Language cache was hit %s time%s" %
(st['cache_hits'], plural(st['cache_hits'])))
logger.debug("Unwanted language removed %s book%s" %
(st['bad_lang'], plural(st['bad_lang'])))
logger.debug("Unwanted characters removed %s book%s" %
(st['bad_char'], plural(st['bad_char'])))
logger.debug("Unable to cache language for %s book%s with missing ISBN" %
(st['uncached'], plural(st['uncached'])))
logger.debug("Found %s duplicate book%s" %
(st['duplicates'], plural(st['duplicates'])))
logger.debug("Rescan %s hit%s, %s miss" %
(rescan_hits, plural(rescan_hits), rescan_count - rescan_hits))
logger.debug("Cache %s hit%s, %s miss" %
(lazylibrarian.CACHE_HIT, plural(lazylibrarian.CACHE_HIT), lazylibrarian.CACHE_MISS))
cachesize = myDB.match("select count('ISBN') as counter from languages")
logger.debug("ISBN Language cache holds %s entries" % cachesize['counter'])
# Cache any covers and images
images = myDB.select('select bookid, bookimg, bookname from books where bookimg like "http%"')
if len(images):
logger.info("Caching cover%s for %i book%s" % (plural(len(images)), len(images), plural(len(images))))
for item in images:
bookid = item['bookid']
bookimg = item['bookimg']
# bookname = item['bookname']
newimg, success = cache_img("book", bookid, bookimg)
if success:
myDB.action('update books set BookImg="%s" where BookID="%s"' % (newimg, bookid))
images = myDB.select('select AuthorID, AuthorImg, AuthorName from authors where AuthorImg like "http%"')
if len(images):
logger.info("Caching image%s for %i author%s" % (plural(len(images)), len(images), plural(len(images))))
for item in images:
authorid = item['authorid']
authorimg = item['authorimg']
# authorname = item['authorname']
newimg, success = cache_img("author", authorid, authorimg)
if success:
myDB.action('update authors set AuthorImg="%s" where AuthorID="%s"' % (newimg, authorid))
# On full scan, update bookcounts for all authors, not just new ones - refresh may have located
# new books for existing authors especially if switched provider gb/gr or changed wanted languages
authors = myDB.select('select AuthorID from authors')
else:
# On single author/book import, just update bookcount for that author
authors = myDB.select('select AuthorID from authors where AuthorName = "%s"' % author.replace('"', '""'))
logger.debug('Updating bookcounts for %i author%s' % (len(authors), plural(len(authors))))
for author in authors:
update_totals(author['AuthorID'])
logger.info('Library scan complete')
return new_book_count
except Exception:
logger.error('Unhandled exception in libraryScan: %s' % traceback.format_exc())