def encode_multipart(fields, files, boundary=None):
"""Encode dict of form fields and dict of files as multipart/form-data.
Return tuple of (body_string, headers_dict). Each value in files is a dict
with required keys 'filename' and 'content', and optional 'mimetype' (if
not specified, tries to guess mime type or uses 'application/octet-stream').
"""
def escape_quote(s):
s = makeUnicode(s)
return s.replace('"', '\\"')
if boundary is None:
boundary = ''.join(random.choice(_BOUNDARY_CHARS) for _ in range(30))
lines = []
if fields:
fields = dict(
(makeBytestr(k), makeBytestr(v)) for k, v in fields.items())
for name, value in list(fields.items()):
lines.extend((
'--{0}'.format(boundary),
'Content-Disposition: form-data; name="{0}"'.format(
escape_quote(name)),
'',
makeUnicode(value),
))
if files:
for name, value in list(files.items()):
filename = value['filename']
if 'mimetype' in value:
mimetype = value['mimetype']
else:
mimetype = mimetypes.guess_type(
filename)[0] or 'application/octet-stream'
lines.extend((
'--{0}'.format(boundary),
'Content-Disposition: form-data; name="{0}"; filename="{1}"'.
format(escape_quote(name), escape_quote(filename)),
'Content-Type: {0}'.format(mimetype),
'',
value['content'],
))
lines.extend((
'--{0}--'.format(boundary),
'',
))
body = makeBytestr('\r\n'.join(lines))
headers = {
'Content-Type': 'multipart/form-data; boundary={0}'.format(boundary),
'Content-Length': str(len(body)),
}
return body, headers
def encode_multipart(fields, files, boundary=None):
"""Encode dict of form fields and dict of files as multipart/form-data.
Return tuple of (body_string, headers_dict). Each value in files is a dict
with required keys 'filename' and 'content', and optional 'mimetype' (if
not specified, tries to guess mime type or uses 'application/octet-stream').
"""
def escape_quote(s):
s = makeUnicode(s)
return s.replace('"', '\\"')
if boundary is None:
boundary = ''.join(random.choice(_BOUNDARY_CHARS) for _ in range(30))
lines = []
if fields:
fields = dict((makeBytestr(k), makeBytestr(v)) for k, v in fields.items())
for name, value in list(fields.items()):
lines.extend((
'--{0}'.format(boundary),
'Content-Disposition: form-data; name="{0}"'.format(escape_quote(name)),
'',
makeUnicode(value),
))
if files:
for name, value in list(files.items()):
filename = value['filename']
if 'mimetype' in value:
mimetype = value['mimetype']
else:
mimetype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
lines.extend((
'--{0}'.format(boundary),
'Content-Disposition: form-data; name="{0}"; filename="{1}"'.format(
escape_quote(name), escape_quote(filename)),
'Content-Type: {0}'.format(mimetype),
'',
value['content'],
))
lines.extend((
'--{0}--'.format(boundary),
'',
))
body = makeBytestr('\r\n'.join(lines))
headers = {
'Content-Type': 'multipart/form-data; boundary={0}'.format(boundary),
'Content-Length': str(len(body)),
}
return body, headers
def dump_table(table, savedir=None, status=None):
myDB = database.DBConnection()
# noinspection PyBroadException
try:
columns = myDB.select('PRAGMA table_info(%s)' % table)
if not columns: # no such table
logger.warn("No such table [%s]" % table)
return 0
if not os.path.isdir(savedir):
savedir = lazylibrarian.DATADIR
headers = ''
for item in columns:
if headers:
headers += ','
headers += item[1]
if status:
cmd = 'SELECT %s from %s WHERE status="%s"' % (headers, table,
status)
else:
cmd = 'SELECT %s from %s' % (headers, table)
data = myDB.select(cmd)
count = 0
if data is not None:
label = table
if status:
label += '_%s' % status
csvFile = os.path.join(savedir, "%s.csv" % label)
if PY2:
fmode = 'wb'
else:
fmode = 'w'
with open(csvFile, fmode) as csvfile:
csvwrite = writer(csvfile,
delimiter=',',
quotechar='"',
quoting=QUOTE_MINIMAL)
headers = headers.split(',')
csvwrite.writerow(headers)
for item in data:
if PY2:
csvwrite.writerow(
[makeBytestr(s) if s else '' for s in item])
else:
csvwrite.writerow([str(s) if s else '' for s in item])
count += 1
msg = "Exported %s item%s to %s" % (count, plural(count), csvFile)
logger.info(msg)
return count
except Exception:
msg = 'Unhandled exception in dump_table: %s' % traceback.format_exc()
logger.error(msg)
return 0
def any_file(search_dir=None, extn=None):
# find a file with specified extension in a directory, any will do
# return full pathname of file, or empty string if none found
if search_dir is None or extn is None:
return ""
if os.path.isdir(search_dir):
for fname in os.listdir(makeBytestr(search_dir)):
fname = makeUnicode(fname)
if fname.endswith(extn):
return os.path.join(search_dir, fname)
return ""
def _get_sid(self, base_url, username, password):
# login so we can capture SID cookie
login_data = makeBytestr(urlencode({'username': username, 'password': password}))
try:
_ = self.opener.open(base_url + '/login', login_data)
except Exception as err:
logger.error('Error getting SID. qBittorrent %s: %s' % (type(err).__name__, str(err)))
logger.warn('Unable to log in to %s/login' % base_url)
return
for cookie in self.cookiejar:
logger.debug('login cookie: ' + cookie.name + ', value: ' + cookie.value)
return
def _get_sid(self, base_url, username, password):
# login so we can capture SID cookie
login_data = makeBytestr(urlencode({'username': username, 'password': password}))
try:
_ = self.opener.open(base_url + '/login', login_data)
except Exception as err:
logger.error('Error getting SID. qBittorrent %s: %s' % (type(err).__name__, str(err)))
logger.warn('Unable to log in to %s/login' % base_url)
return
for cookie in self.cookiejar:
logger.debug('login cookie: ' + cookie.name + ', value: ' + cookie.value)
return
def dump_table(table, savedir=None, status=None):
myDB = database.DBConnection()
# noinspection PyBroadException
try:
columns = myDB.select('PRAGMA table_info(%s)' % table)
if not columns: # no such table
logger.warn("No such table [%s]" % table)
return 0
if not os.path.isdir(savedir):
savedir = lazylibrarian.DATADIR
headers = ''
for item in columns:
if headers:
headers += ','
headers += item[1]
if status:
cmd = 'SELECT %s from %s WHERE status="%s"' % (headers, table, status)
else:
cmd = 'SELECT %s from %s' % (headers, table)
data = myDB.select(cmd)
count = 0
if data is not None:
label = table
if status:
label += '_%s' % status
csvFile = os.path.join(savedir, "%s.csv" % label)
if PY2:
fmode = 'wb'
else:
fmode = 'w'
with open(csvFile, fmode) as csvfile:
csvwrite = writer(csvfile, delimiter=',', quotechar='"', quoting=QUOTE_MINIMAL)
headers = headers.split(',')
csvwrite.writerow(headers)
for item in data:
if PY2:
csvwrite.writerow([makeBytestr(s) if s else '' for s in item])
else:
csvwrite.writerow([str(s) if s else '' for s in item])
count += 1
msg = "Exported %s item%s to %s" % (count, plural(count), csvFile)
logger.info(msg)
return count
except Exception:
msg = 'Unhandled exception in dump_table: %s' % traceback.format_exc()
logger.error(msg)
return 0
def _command(self, command, args=None, content_type=None, files=None):
logger.debug('QBittorrent WebAPI Command: %s' % command)
url = self.base_url + '/' + command
data = None
headers = dict()
if files or content_type == 'multipart/form-data':
data, headers = encode_multipart(
args, files, '-------------------------acebdf13572468')
else:
if args:
data = makeBytestr(urlencode(args))
if content_type:
headers['Content-Type'] = content_type
request = Request(url, data, headers)
if lazylibrarian.CONFIG['PROXY_HOST']:
for item in getList(lazylibrarian.CONFIG['PROXY_TYPE']):
request.set_proxy(lazylibrarian.CONFIG['PROXY_HOST'], item)
request.add_header('User-Agent', USER_AGENT)
try:
response = self.opener.open(request)
try:
contentType = response.headers['content-type']
except KeyError:
contentType = ''
resp = response.read()
# some commands return json
if contentType == 'application/json':
if resp:
return json.loads(resp)
return ''
else:
# some commands return plain text
resp = makeUnicode(resp)
logger.debug("QBitTorrent returned %s" % resp)
if command == 'version/api':
return resp
# some just return Ok. or Fails.
if resp and resp != 'Ok.':
return False
# some commands return nothing but response code (always 200)
return True
except URLError as err:
logger.debug('Failed URL: %s' % url)
logger.debug('QBitTorrent webUI raised the following error: %s' %
err.reason)
return False
def book_file(search_dir=None, booktype=None):
# find a book/mag file in this directory, any book will do
# return full pathname of book/mag, or empty string if none found
if search_dir is None or booktype is None:
return ""
if search_dir and os.path.isdir(search_dir):
try:
for fname in os.listdir(makeBytestr(search_dir)):
fname = makeUnicode(fname)
if is_valid_booktype(fname, booktype=booktype):
return os.path.join(search_dir, fname)
except Exception as e:
logger.warn('Listdir error [%s]: %s %s' % (search_dir, type(e).__name__, str(e)))
return ""
def _command(self, command, args=None, content_type=None, files=None):
logger.debug('QBittorrent WebAPI Command: %s' % command)
url = self.base_url + '/' + command
data = None
headers = dict()
if files or content_type == 'multipart/form-data':
data, headers = encode_multipart(args, files, '-------------------------acebdf13572468')
else:
if args:
data = makeBytestr(urlencode(args))
if content_type:
headers['Content-Type'] = content_type
request = Request(url, data, headers)
if lazylibrarian.CONFIG['PROXY_HOST']:
for item in getList(lazylibrarian.CONFIG['PROXY_TYPE']):
request.set_proxy(lazylibrarian.CONFIG['PROXY_HOST'], item)
request.add_header('User-Agent', getUserAgent())
try:
response = self.opener.open(request)
try:
contentType = response.headers['content-type']
except KeyError:
contentType = ''
resp = response.read()
# some commands return json
if contentType == 'application/json':
if resp:
return json.loads(resp)
return ''
else:
# some commands return plain text
resp = makeUnicode(resp)
logger.debug("QBitTorrent returned %s" % resp)
if command == 'version/api':
return resp
# some just return Ok. or Fails.
if resp and resp != 'Ok.':
return False
# some commands return nothing but response code (always 200)
return True
except URLError as err:
logger.debug('Failed URL: %s' % url)
logger.debug('QBitTorrent webUI raised the following error: %s' % err.reason)
return False
def _listMissingWorkpages(self):
# first the ones with no workpage
q = 'SELECT BookID from books where length(WorkPage) < 4'
res = self._dic_from_query(q)
# now the ones with an error page
cache = os.path.join(lazylibrarian.CACHEDIR, "WorkCache")
if os.path.isdir(cache):
for cached_file in os.listdir(makeBytestr(cache)):
cached_file = makeUnicode(cached_file)
target = os.path.join(cache, cached_file)
if os.path.isfile(target):
if os.path.getsize(target) < 500 and '.' in cached_file:
bookid = cached_file.split('.')[0]
res.append({"BookID": bookid})
self.data = res
def _send_tweet(self, message=None):
username = self.consumer_key
password = self.consumer_secret
access_token_key = lazylibrarian.CONFIG['TWITTER_USERNAME']
access_token_secret = lazylibrarian.CONFIG['TWITTER_PASSWORD']
if not access_token_key or not access_token_secret:
logger.error("No authorization found for twitter")
return False
logger.info("Sending tweet: " + message)
api = twitter.Api(username, password, access_token_key, access_token_secret)
message = formatter.makeBytestr(message)
try:
api.PostUpdate(message)
except Exception as e:
logger.error("Error Sending Tweet: %s" % e)
return False
return True
def _send_tweet(self, message=None):
username = self.consumer_key
password = self.consumer_secret
access_token_key = lazylibrarian.CONFIG['TWITTER_USERNAME']
access_token_secret = lazylibrarian.CONFIG['TWITTER_PASSWORD']
if not access_token_key or not access_token_secret:
logger.error("No authorization found for twitter")
return False
logger.info("Sending tweet: " + message)
api = twitter.Api(username, password, access_token_key,
access_token_secret)
message = formatter.makeBytestr(message)
try:
api.PostUpdate(message)
except Exception as e:
logger.error("Error Sending Tweet: %s" % e)
return False
return True
def opf_file(search_dir=None):
if search_dir is None:
return ""
cnt = 0
res = ''
meta = ''
if os.path.isdir(search_dir):
for fname in os.listdir(makeBytestr(search_dir)):
fname = makeUnicode(fname)
if fname.endswith('.opf'):
if fname == 'metadata.opf':
meta = os.path.join(search_dir, fname)
else:
res = os.path.join(search_dir, fname)
cnt += 1
if cnt > 2 or cnt == 2 and not meta:
logger.debug("Found %d conflicting opf in %s" % (cnt, search_dir))
res = ''
elif res: # prefer bookname.opf over metadata.opf
return res
elif meta:
return meta
return res
def bookRename(bookid):
myDB = database.DBConnection()
cmd = 'select AuthorName,BookName,BookFile from books,authors where books.AuthorID = authors.AuthorID and bookid=?'
exists = myDB.match(cmd, (bookid,))
if not exists:
logger.debug("Invalid bookid in bookRename %s" % bookid)
return ''
f = exists['BookFile']
if not f:
logger.debug("No filename for %s in BookRename %s" % bookid)
return ''
r = os.path.dirname(f)
try:
# noinspection PyTypeChecker
calibreid = r.rsplit('(', 1)[1].split(')')[0]
if not calibreid.isdigit():
calibreid = ''
except IndexError:
calibreid = ''
if calibreid:
msg = '[%s] looks like a calibre directory: not renaming book' % os.path.basename(r)
logger.debug(msg)
return f
seriesinfo = seriesInfo(bookid)
dest_path = lazylibrarian.CONFIG['EBOOK_DEST_FOLDER'].replace(
'$Author', exists['AuthorName']).replace(
'$Title', exists['BookName']).replace(
'$Series', seriesinfo['Full']).replace(
'$SerName', seriesinfo['Name']).replace(
'$SerNum', seriesinfo['Num']).replace(
'$$', ' ')
dest_path = ' '.join(dest_path.split()).strip()
dest_path = replace_all(dest_path, __dic__)
dest_dir = lazylibrarian.DIRECTORY('eBook')
dest_path = os.path.join(dest_dir, dest_path)
if r != dest_path:
try:
dest_path = safe_move(r, dest_path)
r = dest_path
except Exception as why:
if not os.path.isdir(dest_path):
logger.error('Unable to create directory %s: %s' % (dest_path, why))
book_basename, prefextn = os.path.splitext(os.path.basename(f))
new_basename = lazylibrarian.CONFIG['EBOOK_DEST_FILE']
seriesinfo = seriesInfo(bookid)
new_basename = new_basename.replace(
'$Author', exists['AuthorName']).replace(
'$Title', exists['BookName']).replace(
'$Series', seriesinfo['Full']).replace(
'$SerName', seriesinfo['Name']).replace(
'$SerNum', seriesinfo['Num']).replace(
'$$', ' ')
new_basename = ' '.join(new_basename.split()).strip()
# replace all '/' not surrounded by whitespace with '_' as '/' is a directory separator
slash = new_basename.find('/')
while slash > 0:
if new_basename[slash - 1] != ' ':
if new_basename[slash + 1] != ' ':
new_basename = new_basename[:slash] + '_' + new_basename[slash + 1:]
slash = new_basename.find('/', slash + 1)
if ' / ' in new_basename: # used as a separator in goodreads omnibus
logger.warn("bookRename [%s] looks like an omnibus? Not renaming %s" % (new_basename, book_basename))
new_basename = book_basename
if book_basename != new_basename:
# only rename bookname.type, bookname.jpg, bookname.opf, not cover.jpg or metadata.opf
for fname in os.listdir(makeBytestr(r)):
fname = makeUnicode(fname)
extn = ''
if is_valid_booktype(fname, booktype='ebook'):
extn = os.path.splitext(fname)[1]
elif fname.endswith('.opf') and not fname == 'metadata.opf':
extn = '.opf'
elif fname.endswith('.jpg') and not fname == 'cover.jpg':
extn = '.jpg'
if extn:
ofname = os.path.join(r, fname)
nfname = os.path.join(r, new_basename + extn)
try:
nfname = safe_move(ofname, nfname)
logger.debug("bookRename %s to %s" % (ofname, nfname))
if ofname == exists['BookFile']: # if we renamed the preferred filetype, return new name
f = nfname
except Exception as e:
logger.error('Unable to rename [%s] to [%s] %s %s' %
(ofname, nfname, type(e).__name__, str(e)))
return f
def cleanCache():
""" Remove unused files from the cache - delete if expired or unused.
Check JSONCache WorkCache XMLCache SeriesCache Author Book Magazine
Check covers and authorimages referenced in the database exist and change database entry if missing """
myDB = database.DBConnection()
result = []
cache = os.path.join(lazylibrarian.CACHEDIR, "JSONCache")
cleaned = 0
kept = 0
if os.path.isdir(cache):
for cached_file in os.listdir(makeBytestr(cache)):
cached_file = makeUnicode(cached_file)
target = os.path.join(cache, cached_file)
cache_modified_time = os.stat(target).st_mtime
time_now = time.time()
if cache_modified_time < time_now - (
lazylibrarian.CONFIG['CACHE_AGE'] * 24 * 60 * 60): # expire after this many seconds
# Cache is old, delete entry
os.remove(target)
cleaned += 1
else:
kept += 1
msg = "Cleaned %i file%s from JSONCache, kept %i" % (cleaned, plural(cleaned), kept)
result.append(msg)
logger.debug(msg)
cache = os.path.join(lazylibrarian.CACHEDIR, "XMLCache")
cleaned = 0
kept = 0
if os.path.isdir(cache):
for cached_file in os.listdir(makeBytestr(cache)):
cached_file = makeUnicode(cached_file)
target = os.path.join(cache, cached_file)
cache_modified_time = os.stat(target).st_mtime
time_now = time.time()
if cache_modified_time < time_now - (
lazylibrarian.CONFIG['CACHE_AGE'] * 24 * 60 * 60): # expire after this many seconds
# Cache is old, delete entry
os.remove(target)
cleaned += 1
else:
kept += 1
msg = "Cleaned %i file%s from XMLCache, kept %i" % (cleaned, plural(cleaned), kept)
result.append(msg)
logger.debug(msg)
cache = os.path.join(lazylibrarian.CACHEDIR, "WorkCache")
cleaned = 0
kept = 0
if os.path.isdir(cache):
for cached_file in os.listdir(makeBytestr(cache)):
cached_file = makeUnicode(cached_file)
target = os.path.join(cache, cached_file)
try:
bookid = cached_file.split('.')[0]
except IndexError:
logger.error('Clean Cache: Error splitting %s' % cached_file)
continue
item = myDB.match('select BookID from books where BookID=?', (bookid,))
if not item:
# WorkPage no longer referenced in database, delete cached_file
os.remove(target)
cleaned += 1
else:
kept += 1
msg = "Cleaned %i file%s from WorkCache, kept %i" % (cleaned, plural(cleaned), kept)
result.append(msg)
logger.debug(msg)
cache = os.path.join(lazylibrarian.CACHEDIR, "SeriesCache")
cleaned = 0
kept = 0
if os.path.isdir(cache):
for cached_file in os.listdir(makeBytestr(cache)):
cached_file = makeUnicode(cached_file)
target = os.path.join(cache, cached_file)
try:
seriesid = cached_file.split('.')[0]
except IndexError:
logger.error('Clean Cache: Error splitting %s' % cached_file)
continue
item = myDB.match('select SeriesID from series where SeriesID=?', (seriesid,))
if not item:
# SeriesPage no longer referenced in database, delete cached_file
os.remove(target)
cleaned += 1
else:
kept += 1
msg = "Cleaned %i file%s from SeriesCache, kept %i" % (cleaned, plural(cleaned), kept)
result.append(msg)
logger.debug(msg)
cache = os.path.join(lazylibrarian.CACHEDIR, "magazine")
cleaned = 0
kept = 0
if os.path.isdir(cache):
# we can clear the magazine cache, it gets rebuilt as required
# this does not delete our magazine cover files, only the small cached copy
for cached_file in os.listdir(makeBytestr(cache)):
cached_file = makeUnicode(cached_file)
target = os.path.join(cache, cached_file)
if target.endswith('.jpg'):
os.remove(target)
cleaned += 1
else:
kept += 1
msg = "Cleaned %i file%s from magazine cache, kept %i" % (cleaned, plural(cleaned), kept)
result.append(msg)
logger.debug(msg)
cache = lazylibrarian.CACHEDIR
cleaned = 0
kept = 0
cachedir = os.path.join(cache, 'author')
if os.path.isdir(cachedir):
for cached_file in os.listdir(makeBytestr(cachedir)):
cached_file = makeUnicode(cached_file)
target = os.path.join(cachedir, cached_file)
if os.path.isfile(target):
try:
imgid = cached_file.split('.')[0].rsplit(os.sep)[-1]
except IndexError:
logger.error('Clean Cache: Error splitting %s' % cached_file)
continue
item = myDB.match('select AuthorID from authors where AuthorID=?', (imgid,))
if not item:
# Author Image no longer referenced in database, delete cached_file
os.remove(target)
cleaned += 1
else:
kept += 1
cachedir = os.path.join(cache, 'book')
if os.path.isdir(cachedir):
for cached_file in os.listdir(makeBytestr(cachedir)):
cached_file = makeUnicode(cached_file)
target = os.path.join(cachedir, cached_file)
if os.path.isfile(target):
try:
imgid = cached_file.split('.')[0].rsplit(os.sep)[-1]
except IndexError:
logger.error('Clean Cache: Error splitting %s' % cached_file)
continue
item = myDB.match('select BookID from books where BookID=?', (imgid,))
if not item:
# Book Image no longer referenced in database, delete cached_file
os.remove(target)
cleaned += 1
else:
kept += 1
# at this point there should be no more .jpg files in the root of the cachedir
# any that are still there are for books/authors deleted from database
for cached_file in os.listdir(makeBytestr(cache)):
cached_file = makeUnicode(cached_file)
if cached_file.endswith('.jpg'):
os.remove(os.path.join(cache, cached_file))
cleaned += 1
msg = "Cleaned %i file%s from ImageCache, kept %i" % (cleaned, plural(cleaned), kept)
result.append(msg)
logger.debug(msg)
# verify the cover images referenced in the database are present
images = myDB.action('select BookImg,BookName,BookID from books')
cachedir = os.path.join(lazylibrarian.CACHEDIR, 'book')
cleaned = 0
kept = 0
for item in images:
keep = True
imgfile = ''
if item['BookImg'] is None or item['BookImg'] == '':
keep = False
if keep and not item['BookImg'].startswith('http') and not item['BookImg'] == "images/nocover.png":
# html uses '/' as separator, but os might not
imgname = item['BookImg'].rsplit('/')[-1]
imgfile = os.path.join(cachedir, imgname)
if not os.path.isfile(imgfile):
keep = False
if keep:
kept += 1
else:
cleaned += 1
logger.debug('Cover missing for %s %s' % (item['BookName'], imgfile))
myDB.action('update books set BookImg="images/nocover.png" where Bookid=?', (item['BookID'],))
msg = "Cleaned %i missing cover file%s, kept %i" % (cleaned, plural(cleaned), kept)
result.append(msg)
logger.debug(msg)
# verify the author images referenced in the database are present
images = myDB.action('select AuthorImg,AuthorName,AuthorID from authors')
cachedir = os.path.join(lazylibrarian.CACHEDIR, 'author')
cleaned = 0
kept = 0
for item in images:
keep = True
imgfile = ''
if item['AuthorImg'] is None or item['AuthorImg'] == '':
keep = False
if keep and not item['AuthorImg'].startswith('http') and not item['AuthorImg'] == "images/nophoto.png":
# html uses '/' as separator, but os might not
imgname = item['AuthorImg'].rsplit('/')[-1]
imgfile = os.path.join(cachedir, imgname)
if not os.path.isfile(imgfile):
keep = False
if keep:
kept += 1
else:
cleaned += 1
logger.debug('Image missing for %s %s' % (item['AuthorName'], imgfile))
myDB.action('update authors set AuthorImg="images/nophoto.png" where AuthorID=?', (item['AuthorID'],))
msg = "Cleaned %i missing author image%s, kept %i" % (cleaned, plural(cleaned), kept)
result.append(msg)
logger.debug(msg)
return result
def bookRename(bookid):
myDB = database.DBConnection()
cmd = 'select AuthorName,BookName,BookFile from books,authors where books.AuthorID = authors.AuthorID and bookid=?'
exists = myDB.match(cmd, (bookid,))
if not exists:
logger.debug("Invalid bookid in bookRename %s" % bookid)
return ''
f = exists['BookFile']
if not f:
logger.debug("No filename for %s in BookRename %s" % bookid)
return ''
r = os.path.dirname(f)
if not lazylibrarian.CONFIG['CALIBRE_RENAME']:
try:
# noinspection PyTypeChecker
calibreid = r.rsplit('(', 1)[1].split(')')[0]
if not calibreid.isdigit():
calibreid = ''
except IndexError:
calibreid = ''
if calibreid:
msg = '[%s] looks like a calibre directory: not renaming book' % os.path.basename(r)
logger.debug(msg)
return f
reject = multibook(r)
if reject:
logger.debug("Not renaming %s, found multiple %s" % (f, reject))
return f
seriesinfo = nameVars(bookid)
dest_path = seriesinfo['FolderName']
dest_dir = lazylibrarian.DIRECTORY('eBook')
dest_path = os.path.join(dest_dir, dest_path)
dest_path = stripspaces(dest_path)
oldpath = r
if oldpath != dest_path:
try:
dest_path = safe_move(oldpath, dest_path)
except Exception as why:
if not os.path.isdir(dest_path):
logger.error('Unable to create directory %s: %s' % (dest_path, why))
book_basename, prefextn = os.path.splitext(os.path.basename(f))
new_basename = seriesinfo['BookFile']
if ' / ' in new_basename: # used as a separator in goodreads omnibus
logger.warn("bookRename [%s] looks like an omnibus? Not renaming %s" % (new_basename, book_basename))
new_basename = book_basename
if book_basename != new_basename:
# only rename bookname.type, bookname.jpg, bookname.opf, not cover.jpg or metadata.opf
for fname in os.listdir(makeBytestr(dest_path)):
fname = makeUnicode(fname)
extn = ''
if is_valid_booktype(fname, booktype='ebook'):
extn = os.path.splitext(fname)[1]
elif fname.endswith('.opf') and not fname == 'metadata.opf':
extn = '.opf'
elif fname.endswith('.jpg') and not fname == 'cover.jpg':
extn = '.jpg'
if extn:
ofname = os.path.join(dest_path, fname)
nfname = os.path.join(dest_path, new_basename + extn)
if ofname != nfname:
try:
nfname = safe_move(ofname, nfname)
logger.debug("bookRename %s to %s" % (ofname, nfname))
oldname = os.path.join(oldpath, fname)
if oldname == exists['BookFile']: # if we renamed/moved the preferred file, return new name
f = nfname
except Exception as e:
logger.error('Unable to rename [%s] to [%s] %s %s' %
(ofname, nfname, type(e).__name__, str(e)))
return f
def audioProcess(bookid, rename=False, playlist=False):
"""
:param bookid: book to process
:param rename: rename to match audiobook filename pattern
:param playlist: generate a playlist for popup
:return: filename of part 01 of the audiobook
"""
for item in ['$Part', '$Title']:
if rename and item not in lazylibrarian.CONFIG['AUDIOBOOK_DEST_FILE']:
logger.error("Unable to audioProcess, check AUDIOBOOK_DEST_FILE")
return ''
myDB = database.DBConnection()
cmd = 'select AuthorName,BookName,AudioFile from books,authors where books.AuthorID = authors.AuthorID and bookid=?'
exists = myDB.match(cmd, (bookid,))
if exists:
book_filename = exists['AudioFile']
if book_filename:
r = os.path.dirname(book_filename)
else:
logger.debug("No filename for %s in audioProcess" % bookid)
return ''
else:
logger.debug("Invalid bookid in audioProcess %s" % bookid)
return ''
if not TinyTag:
logger.warn("TinyTag library not available")
return ''
cnt = 0
parts = []
total = 0
author = ''
book = ''
audio_file = ''
abridged = ''
for f in os.listdir(makeBytestr(r)):
f = makeUnicode(f)
if is_valid_booktype(f, booktype='audiobook'):
cnt += 1
audio_file = f
try:
audio_path = os.path.join(r, f)
performer = ''
composer = ''
albumartist = ''
book = ''
title = ''
track = 0
total = 0
if TinyTag.is_supported(audio_path):
id3r = TinyTag.get(audio_path)
performer = id3r.artist
composer = id3r.composer
albumartist = id3r.albumartist
book = id3r.album
title = id3r.title
track = id3r.track
total = id3r.track_total
track = check_int(track, 0)
total = check_int(total, 0)
if performer:
performer = performer.strip()
if composer:
composer = composer.strip()
if book:
book = book.strip()
if albumartist:
albumartist = albumartist.strip()
if composer: # if present, should be author
author = composer
elif performer: # author, or narrator if composer == author
author = performer
elif albumartist:
author = albumartist
if author and book:
parts.append([track, book, author, f])
if not abridged:
for tag in [book, title, albumartist, performer, composer]:
if tag and 'unabridged' in tag.lower():
abridged = 'Unabridged'
break
if not abridged:
for tag in [book, title, albumartist, performer, composer]:
if tag and 'abridged' in tag.lower():
abridged = 'Abridged'
break
except Exception as e:
logger.error("tinytag %s %s" % (type(e).__name__, str(e)))
pass
finally:
if not abridged:
if audio_file and 'unabridged' in audio_file.lower():
abridged = 'Unabridged'
break
if not abridged:
if audio_file and 'abridged' in audio_file.lower():
abridged = 'Abridged'
break
logger.debug("%s found %s audiofile%s" % (exists['BookName'], cnt, plural(cnt)))
if cnt == 1 and not parts: # single file audiobook with no tags
parts = [[1, exists['BookName'], exists['AuthorName'], audio_file]]
if cnt != len(parts):
logger.warn("%s: Incorrect number of parts (found %i from %i)" % (exists['BookName'], len(parts), cnt))
return book_filename
if total and total != cnt:
logger.warn("%s: Reported %i parts, got %i" % (exists['BookName'], total, cnt))
return book_filename
# check all parts have the same author and title
if len(parts) > 1:
for part in parts:
if part[1] != book:
logger.warn("%s: Inconsistent title: [%s][%s]" % (exists['BookName'], part[1], book))
return book_filename
if part[2] != author:
logger.warn("%s: Inconsistent author: [%s][%s]" % (exists['BookName'], part[2], author))
return book_filename
# do we have any track info (value is 0 if not)
if parts[0][0] == 0:
tokmatch = ''
# try to extract part information from filename. Search for token style of part 1 in this order...
for token in [' 001.', ' 01.', ' 1.', ' 001 ', ' 01 ', ' 1 ', '01']:
if tokmatch:
break
for part in parts:
if token in part[3]:
tokmatch = token
break
if tokmatch: # we know the numbering style, get numbers for the other parts
cnt = 0
while cnt < len(parts):
cnt += 1
if tokmatch == ' 001.':
pattern = ' %s.' % str(cnt).zfill(3)
elif tokmatch == ' 01.':
pattern = ' %s.' % str(cnt).zfill(2)
elif tokmatch == ' 1.':
pattern = ' %s.' % str(cnt)
elif tokmatch == ' 001 ':
pattern = ' %s ' % str(cnt).zfill(3)
elif tokmatch == ' 01 ':
pattern = ' %s ' % str(cnt).zfill(2)
elif tokmatch == ' 1 ':
pattern = ' %s ' % str(cnt)
else:
pattern = '%s' % str(cnt).zfill(2)
# standardise numbering of the parts
for part in parts:
if pattern in part[3]:
part[0] = cnt
break
parts.sort(key=lambda x: x[0])
# check all parts are present
cnt = 0
while cnt < len(parts):
if parts[cnt][0] != cnt + 1:
logger.warn("%s: No part %i found" % (exists['BookName'], cnt + 1))
return book_filename
cnt += 1
if abridged:
abridged = ' (%s)' % abridged
# if we get here, looks like we have all the parts needed to rename properly
seriesinfo = nameVars(bookid, abridged)
dest_path = seriesinfo['FolderName']
dest_dir = lazylibrarian.DIRECTORY('Audio')
dest_path = os.path.join(dest_dir, dest_path)
if rename and r != dest_path:
try:
dest_path = safe_move(r, dest_path)
r = dest_path
except Exception as why:
if not os.path.isdir(dest_path):
logger.error('Unable to create directory %s: %s' % (dest_path, why))
if playlist:
try:
playlist = open(os.path.join(r, 'playlist.ll'), 'w')
except Exception as why:
logger.error('Unable to create playlist in %s: %s' % (r, why))
playlist = None
for part in parts:
pattern = seriesinfo['AudioFile']
pattern = pattern.replace(
'$Part', str(part[0]).zfill(len(str(len(parts))))).replace(
'$Total', str(len(parts)))
pattern = ' '.join(pattern.split()).strip()
pattern = pattern + os.path.splitext(part[3])[1]
if playlist:
if rename:
playlist.write(pattern + '\n')
else:
playlist.write(part[3] + '\n')
if rename:
n = os.path.join(r, pattern)
o = os.path.join(r, part[3])
if o != n:
try:
n = safe_move(o, n)
if part[0] == 1:
book_filename = n # return part 1 of set
logger.debug('%s: audioProcess [%s] to [%s]' % (exists['BookName'], o, n))
except Exception as e:
logger.error('Unable to rename [%s] to [%s] %s %s' % (o, n, type(e).__name__, str(e)))
if playlist:
playlist.close()
return book_filename
def createMagCover(issuefile=None, refresh=False):
if not lazylibrarian.CONFIG['IMP_MAGCOVER']:
return
if issuefile is None or not os.path.isfile(issuefile):
logger.debug('No issuefile %s' % issuefile)
return
base, extn = os.path.splitext(issuefile)
if not extn:
logger.debug('Unable to create cover for %s, no extension?' % issuefile)
return
coverfile = base + '.jpg'
if os.path.isfile(coverfile):
if refresh:
os.remove(coverfile)
else:
logger.debug('Cover for %s exists' % issuefile)
return # quit if cover already exists and we didn't want to refresh
logger.debug('Creating cover for %s' % issuefile)
data = '' # result from unzip or unrar
extn = extn.lower()
if extn in ['.cbz', '.epub']:
try:
data = zipfile.ZipFile(issuefile)
except Exception as why:
logger.error("Failed to read zip file %s, %s %s" % (issuefile, type(why).__name__, str(why)))
data = ''
elif extn in ['.cbr']:
try:
# unrar will complain if the library isn't installed, needs to be compiled separately
# see https://pypi.python.org/pypi/unrar/ for instructions
# Download source from http://www.rarlab.com/rar_add.htm
# note we need LIBRARY SOURCE not a binary package
# make lib; sudo make install-lib; sudo ldconfig
# lib.unrar should then be able to find libunrar.so
from lib.unrar import rarfile
data = rarfile.RarFile(issuefile)
except Exception as why:
logger.error("Failed to read rar file %s, %s %s" % (issuefile, type(why).__name__, str(why)))
data = ''
if data:
img = None
try:
for member in data.namelist():
memlow = member.lower()
if '-00.' in memlow or '000.' in memlow or 'cover.' in memlow:
if memlow.endswith('.jpg') or memlow.endswith('.jpeg'):
img = data.read(member)
break
if img:
with open(coverfile, 'wb') as f:
if PY2:
f.write(img)
else:
f.write(img.encode())
return
else:
logger.debug("Failed to find image in %s" % issuefile)
except Exception as why:
logger.error("Failed to extract image from %s, %s %s" % (issuefile, type(why).__name__, str(why)))
elif extn == '.pdf':
generator = ""
if len(lazylibrarian.CONFIG['IMP_CONVERT']): # allow external convert to override libraries
generator = "external program: %s" % lazylibrarian.CONFIG['IMP_CONVERT']
if "gsconvert.py" in lazylibrarian.CONFIG['IMP_CONVERT']:
msg = "Use of gsconvert.py is deprecated, equivalent functionality is now built in. "
msg += "Support for gsconvert.py may be removed in a future release. See wiki for details."
logger.warn(msg)
converter = lazylibrarian.CONFIG['IMP_CONVERT']
postfix = ''
# if not os.path.isfile(converter): # full path given, or just program_name?
# converter = os.path.join(os.getcwd(), lazylibrarian.CONFIG['IMP_CONVERT'])
if 'convert' in converter and 'gs' not in converter:
# tell imagemagick to only convert first page
postfix = '[0]'
try:
params = [converter, '%s%s' % (issuefile, postfix), '%s' % coverfile]
res = subprocess.check_output(params, stderr=subprocess.STDOUT)
res = makeUnicode(res).strip()
if res:
logger.debug('%s reports: %s' % (lazylibrarian.CONFIG['IMP_CONVERT'], res))
except Exception as e:
# logger.debug(params)
logger.warn('External "convert" failed %s %s' % (type(e).__name__, str(e)))
elif platform.system() == "Windows":
GS = os.path.join(os.getcwd(), "gswin64c.exe")
generator = "local gswin64c"
if not os.path.isfile(GS):
GS = os.path.join(os.getcwd(), "gswin32c.exe")
generator = "local gswin32c"
if not os.path.isfile(GS):
params = ["where", "gswin64c"]
try:
GS = subprocess.check_output(params, stderr=subprocess.STDOUT)
GS = makeUnicode(GS).strip()
generator = "gswin64c"
except Exception as e:
logger.debug("where gswin64c failed: %s %s" % (type(e).__name__, str(e)))
if not os.path.isfile(GS):
params = ["where", "gswin32c"]
try:
GS = subprocess.check_output(params, stderr=subprocess.STDOUT)
GS = makeUnicode(GS).strip()
generator = "gswin32c"
except Exception as e:
logger.debug("where gswin32c failed: %s %s" % (type(e).__name__, str(e)))
if not os.path.isfile(GS):
logger.debug("No gswin found")
generator = "(no windows ghostscript found)"
else:
# noinspection PyBroadException
try:
params = [GS, "--version"]
res = subprocess.check_output(params, stderr=subprocess.STDOUT)
res = makeUnicode(res).strip()
logger.debug("Found %s [%s] version %s" % (generator, GS, res))
generator = "%s version %s" % (generator, res)
issuefile = issuefile.split('[')[0]
params = [GS, "-sDEVICE=jpeg", "-dNOPAUSE", "-dBATCH", "-dSAFER", "-dFirstPage=1", "-dLastPage=1",
"-dUseCropBox", "-sOutputFile=%s" % coverfile, issuefile]
res = subprocess.check_output(params, stderr=subprocess.STDOUT)
res = makeUnicode(res).strip()
if not os.path.isfile(coverfile):
logger.debug("Failed to create jpg: %s" % res)
except Exception: # as why:
logger.warn("Failed to create jpg for %s" % issuefile)
logger.debug('Exception in gswin create_cover: %s' % traceback.format_exc())
else: # not windows
try:
# noinspection PyUnresolvedReferences
from wand.image import Image
interface = "wand"
except ImportError:
try:
# No PythonMagick in python3
# noinspection PyUnresolvedReferences
import PythonMagick
interface = "pythonmagick"
except ImportError:
interface = ""
try:
if interface == 'wand':
generator = "wand interface"
with Image(filename=issuefile + '[0]') as img:
img.save(filename=coverfile)
elif interface == 'pythonmagick':
generator = "pythonmagick interface"
img = PythonMagick.Image()
# PythonMagick requires filenames to be bytestr, not unicode
if type(issuefile) is text_type:
issuefile = makeBytestr(issuefile)
if type(coverfile) is text_type:
coverfile = makeBytestr(coverfile)
img.read(issuefile + '[0]')
img.write(coverfile)
else:
GS = os.path.join(os.getcwd(), "gs")
generator = "local gs"
if not os.path.isfile(GS):
GS = ""
params = ["which", "gs"]
try:
GS = subprocess.check_output(params, stderr=subprocess.STDOUT)
GS = makeUnicode(GS).strip()
generator = GS
except Exception as e:
logger.debug("which gs failed: %s %s" % (type(e).__name__, str(e)))
if not os.path.isfile(GS):
logger.debug("Cannot find gs")
generator = "(no gs found)"
else:
params = [GS, "--version"]
res = subprocess.check_output(params, stderr=subprocess.STDOUT)
res = makeUnicode(res).strip()
logger.debug("Found gs [%s] version %s" % (GS, res))
generator = "%s version %s" % (generator, res)
issuefile = issuefile.split('[')[0]
params = [GS, "-sDEVICE=jpeg", "-dNOPAUSE", "-dBATCH", "-dSAFER", "-dFirstPage=1",
"-dLastPage=1", "-dUseCropBox", "-sOutputFile=%s" % coverfile, issuefile]
res = subprocess.check_output(params, stderr=subprocess.STDOUT)
res = makeUnicode(res).strip()
if not os.path.isfile(coverfile):
logger.debug("Failed to create jpg: %s" % res)
except Exception as e:
logger.warn("Unable to create cover for %s using %s %s" % (issuefile, type(e).__name__, generator))
logger.debug('Exception in create_cover: %s' % traceback.format_exc())
if os.path.isfile(coverfile):
setperm(coverfile)
logger.debug("Created cover for %s using %s" % (issuefile, generator))
return
# if not recognised extension or cover creation failed
try:
coverfile = safe_copy(os.path.join(lazylibrarian.PROG_DIR, 'data/images/nocover.jpg'), coverfile)
setperm(coverfile)
except Exception as why:
logger.error("Failed to copy nocover file, %s %s" % (type(why).__name__, str(why)))
return
def create_id(issuename=None):
hashID = sha1(makeBytestr(issuename)).hexdigest()
# logger.debug('Issue %s Hash: %s' % (issuename, hashID))
return hashID
def db_v14(myDB, upgradelog):
upgradelog.write("%s v14: %s\n" % (time.ctime(), "Moving image caches"))
src = lazylibrarian.CACHEDIR
try:
os.mkdir(os.path.join(src, 'author'))
except OSError as e:
if e.errno not in [17, 183]: # already exists is ok
msg = 'mkdir author cache reports: %s' % str(e)
logger.debug(msg)
upgradelog.write("%s v14: %s\n" % (time.ctime(), msg))
query = 'SELECT AuthorName, AuthorID, AuthorImg FROM authors '
query += 'WHERE AuthorImg LIKE "cache/%" '
query += 'AND AuthorImg NOT LIKE "cache/author/%"'
images = myDB.select(query)
if images:
tot = len(images)
msg = 'Moving %s author images to new location' % tot
logger.debug(msg)
upgradelog.write("%s v14: %s\n" % (time.ctime(), msg))
cnt = 0
for image in images:
cnt += 1
lazylibrarian.UPDATE_MSG = "Moving author images to new location: %s of %s" % (cnt, tot)
try:
img = image['AuthorImg']
img = img.rsplit('/', 1)[1]
srcfile = os.path.join(src, img)
if os.path.isfile(srcfile):
try:
shutil.move(srcfile, os.path.join(src, "author", img))
myDB.action(
'UPDATE authors SET AuthorImg="cache/author/?" WHERE AuthorID=?',
(img, image['AuthorID']))
except Exception as e:
logger.error("dbupgrade: %s %s" % (type(e).__name__, str(e)))
except Exception as e:
msg = 'Failed to update author image for %s: %s %s' % (image['AuthorName'], type(e).__name__, str(e))
logger.warn(msg)
upgradelog.write("%s v14: %s\n" % (time.ctime(), msg))
upgradelog.write("%s v14: %s\n" % (time.ctime(), lazylibrarian.UPDATE_MSG))
logger.debug("Author Image cache updated")
try:
os.mkdir(os.path.join(src, 'book'))
except OSError as e:
if e.errno not in [17, 183]: # already exists is ok
msg = 'mkdir book cache reports: %s' % str(e)
logger.debug(msg)
upgradelog.write("%s v14: %s\n" % (time.ctime(), msg))
query = 'SELECT BookName, BookID, BookImg FROM books '
query += 'WHERE BookImg LIKE "cache/%" '
query += 'AND BookImg NOT LIKE "cache/book/%"'
images = myDB.select(query)
if images:
tot = len(images)
msg = 'Moving %s book images to new location' % tot
upgradelog.write("%s v14: %s\n" % (time.ctime(), msg))
logger.debug(msg)
cnt = 0
for image in images:
cnt += 1
lazylibrarian.UPDATE_MSG = "Moving book images to new location: %s of %s" % (cnt, tot)
try:
img = image['BookImg']
img = img.rsplit('/', 1)[1]
srcfile = os.path.join(src, img)
if os.path.isfile(srcfile):
try:
shutil.move(srcfile, os.path.join(src, "book", img))
myDB.action('UPDATE books SET BookImg="cache/book/?" WHERE BookID=?',
(img, image['BookID']))
except Exception as e:
logger.error("dbupgrade: %s %s" % (type(e).__name__, str(e)))
except Exception as e:
logger.warn('Failed to update book image for %s: %s %s' %
(image['BookName'], type(e).__name__, str(e)))
upgradelog.write("%s v14: %s\n" % (time.ctime(), lazylibrarian.UPDATE_MSG))
logger.debug("Book Image cache updated")
# at this point there should be no more .jpg files in the root of the cachedir
# any that are still there are for books/authors deleted from database
# or magazine latest issue cover files that get copied as required
for image in os.listdir(makeBytestr(src)):
image = makeUnicode(image)
if image.endswith('.jpg'):
os.remove(os.path.join(src, image))
upgradelog.write("%s v14: complete\n" % time.ctime())
def magazineScan(title=None):
lazylibrarian.MAG_UPDATE = 1
# noinspection PyBroadException
try:
myDB = database.DBConnection()
onetitle = title
if onetitle:
mag_path = lazylibrarian.CONFIG['MAG_DEST_FOLDER'].replace(
'$Title', onetitle)
else:
mag_path = os.path.dirname(lazylibrarian.CONFIG['MAG_DEST_FOLDER'])
if lazylibrarian.CONFIG['MAG_RELATIVE']:
mag_path = os.path.join(lazylibrarian.DIRECTORY('eBook'), mag_path)
if PY2:
mag_path = mag_path.encode(lazylibrarian.SYS_ENCODING)
if lazylibrarian.CONFIG['FULL_SCAN'] and not onetitle:
mags = myDB.select('select * from Issues')
# check all the issues are still there, delete entry if not
for mag in mags:
title = mag['Title']
issuedate = mag['IssueDate']
issuefile = mag['IssueFile']
if issuefile and not os.path.isfile(issuefile):
myDB.action('DELETE from Issues where issuefile=?',
(issuefile, ))
logger.info('Issue %s - %s deleted as not found on disk' %
(title, issuedate))
controlValueDict = {"Title": title}
newValueDict = {
"LastAcquired": None, # clear magazine dates
"IssueDate": None, # we will fill them in again later
"LatestCover": None,
"IssueStatus":
"Skipped" # assume there are no issues now
}
myDB.upsert("magazines", newValueDict, controlValueDict)
logger.debug('Magazine %s details reset' % title)
# now check the magazine titles and delete any with no issues
if lazylibrarian.CONFIG['MAG_DELFOLDER']:
mags = myDB.select(
'SELECT Title,count(Title) as counter from issues group by Title'
)
for mag in mags:
title = mag['Title']
issues = mag['counter']
if not issues:
logger.debug('Magazine %s deleted as no issues found' %
title)
myDB.action('DELETE from magazines WHERE Title=?',
(title, ))
logger.info(' Checking [%s] for magazines' % mag_path)
matchString = ''
for char in lazylibrarian.CONFIG['MAG_DEST_FILE']:
matchString = matchString + '\\' + char
# massage the MAG_DEST_FILE config parameter into something we can use
# with regular expression matching
booktypes = ''
count = -1
booktype_list = getList(lazylibrarian.CONFIG['MAG_TYPE'])
for book_type in booktype_list:
count += 1
if count == 0:
booktypes = book_type
else:
booktypes = booktypes + '|' + book_type
match = matchString.replace(
"\\$\\I\\s\\s\\u\\e\\D\\a\\t\\e", "(?P<issuedate>.*?)").replace(
"\\$\\T\\i\\t\\l\\e",
"(?P<title>.*?)") + '\.[' + booktypes + ']'
title_pattern = re.compile(match, re.VERBOSE)
match = matchString.replace(
"\\$\\I\\s\\s\\u\\e\\D\\a\\t\\e", "(?P<issuedate>.*?)").replace(
"\\$\\T\\i\\t\\l\\e", "") + '\.[' + booktypes + ']'
date_pattern = re.compile(match, re.VERBOSE)
# try to ensure startdir is str as os.walk can fail if it tries to convert a subdir or file
# to utf-8 and fails (eg scandinavian characters in ascii 8bit)
for rootdir, dirnames, filenames in os.walk(makeBytestr(mag_path)):
rootdir = makeUnicode(rootdir)
filenames = [makeUnicode(item) for item in filenames]
for fname in filenames:
# maybe not all magazines will be pdf?
if is_valid_booktype(fname, booktype='mag'):
issuedate = ''
# noinspection PyBroadException
try:
match = title_pattern.match(fname)
if match:
title = match.group("title")
issuedate = match.group("issuedate")
if lazylibrarian.LOGLEVEL & lazylibrarian.log_magdates:
logger.debug("Title pattern [%s][%s]" %
(title, issuedate))
match = True
else:
logger.debug(
"Title pattern match failed for [%s]" % fname)
except Exception:
match = False
if not match:
# noinspection PyBroadException
try:
match = date_pattern.match(fname)
if match:
issuedate = match.group("issuedate")
title = os.path.basename(rootdir)
if lazylibrarian.LOGLEVEL & lazylibrarian.log_magdates:
logger.debug("Date pattern [%s][%s]" %
(title, issuedate))
match = True
else:
logger.debug(
"Date pattern match failed for [%s]" %
fname)
except Exception:
match = False
if not match:
title = os.path.basename(rootdir)
issuedate = ''
dic = {
'.': ' ',
'-': ' ',
'/': ' ',
'+': ' ',
'_': ' ',
'(': '',
')': '',
'[': ' ',
']': ' ',
'#': '# '
}
if issuedate:
exploded = replace_all(issuedate, dic).split()
regex_pass, issuedate, year = lazylibrarian.searchmag.get_issue_date(
exploded)
if lazylibrarian.LOGLEVEL & lazylibrarian.log_magdates:
logger.debug("Date regex [%s][%s][%s]" %
(regex_pass, issuedate, year))
if not regex_pass:
issuedate = ''
if not issuedate:
exploded = replace_all(fname, dic).split()
regex_pass, issuedate, year = lazylibrarian.searchmag.get_issue_date(
exploded)
if lazylibrarian.LOGLEVEL & lazylibrarian.log_magdates:
logger.debug("File regex [%s][%s][%s]" %
(regex_pass, issuedate, year))
if not regex_pass:
issuedate = ''
if not issuedate:
logger.warn("Invalid name format for [%s]" % fname)
continue
issuefile = os.path.join(rootdir,
fname) # full path to issue.pdf
mtime = os.path.getmtime(issuefile)
iss_acquired = datetime.date.isoformat(
datetime.date.fromtimestamp(mtime))
if lazylibrarian.CONFIG['MAG_RENAME']:
filedate = issuedate
if issuedate and issuedate.isdigit():
if len(issuedate) == 8:
if check_year(issuedate[:4]):
filedate = 'Issue %d %s' % (int(
issuedate[4:]), issuedate[:4])
else:
filedate = 'Vol %d Iss %d' % (int(
issuedate[:4]), int(issuedate[4:]))
elif len(issuedate) == 12:
filedate = 'Vol %d Iss %d %s' % (int(
issuedate[4:8]), int(
issuedate[8:]), issuedate[:4])
else:
filedate = str(issuedate).zfill(4)
extn = os.path.splitext(fname)[1]
newfname = lazylibrarian.CONFIG[
'MAG_DEST_FILE'].replace('$Title', title).replace(
'$IssueDate', filedate)
newfname = newfname + extn
if newfname and newfname != fname:
logger.debug("Rename %s -> %s" % (fname, newfname))
newissuefile = os.path.join(rootdir, newfname)
newissuefile = safe_move(issuefile, newissuefile)
if os.path.exists(issuefile.replace(extn, '.jpg')):
safe_move(issuefile.replace(extn, '.jpg'),
newissuefile.replace(extn, '.jpg'))
if os.path.exists(issuefile.replace(extn, '.opf')):
safe_move(issuefile.replace(extn, '.opf'),
newissuefile.replace(extn, '.opf'))
issuefile = newissuefile
logger.debug("Found %s Issue %s" % (title, issuedate))
controlValueDict = {"Title": title}
# is this magazine already in the database?
mag_entry = myDB.match(
'SELECT LastAcquired,IssueDate,MagazineAdded,CoverPage from magazines WHERE Title=?',
(title, ))
if not mag_entry:
# need to add a new magazine to the database
newValueDict = {
"Reject": None,
"Status": "Active",
"MagazineAdded": None,
"LastAcquired": None,
"LatestCover": None,
"IssueDate": None,
"IssueStatus": "Skipped",
"Regex": None,
"CoverPage": 1
}
logger.debug("Adding magazine %s" % title)
myDB.upsert("magazines", newValueDict,
controlValueDict)
magissuedate = None
magazineadded = None
maglastacquired = None
magcoverpage = 1
else:
maglastacquired = mag_entry['LastAcquired']
magissuedate = mag_entry['IssueDate']
magazineadded = mag_entry['MagazineAdded']
magissuedate = str(magissuedate).zfill(4)
magcoverpage = mag_entry['CoverPage']
issuedate = str(issuedate).zfill(
4) # for sorting issue numbers
# is this issue already in the database?
issue_id = create_id("%s %s" % (title, issuedate))
iss_entry = myDB.match(
'SELECT Title,IssueFile from issues WHERE Title=? and IssueDate=?',
(title, issuedate))
new_entry = False
if not iss_entry or iss_entry['IssueFile'] != issuefile:
new_entry = True # new entry or name changed
if not iss_entry:
logger.debug("Adding issue %s %s" %
(title, issuedate))
else:
logger.debug("Updating issue %s %s" %
(title, issuedate))
controlValueDict = {
"Title": title,
"IssueDate": issuedate
}
newValueDict = {
"IssueAcquired": iss_acquired,
"IssueID": issue_id,
"IssueFile": issuefile
}
myDB.upsert("Issues", newValueDict, controlValueDict)
ignorefile = os.path.join(os.path.dirname(issuefile),
'.ll_ignore')
with open(ignorefile, 'a'):
os.utime(ignorefile, None)
createMagCover(issuefile,
pagenum=magcoverpage,
refresh=new_entry)
lazylibrarian.postprocess.processMAGOPF(
issuefile,
title,
issuedate,
issue_id,
overwrite=new_entry)
# see if this issues date values are useful
controlValueDict = {"Title": title}
if not mag_entry: # new magazine, this is the only issue
newValueDict = {
"MagazineAdded": iss_acquired,
"LastAcquired": iss_acquired,
"LatestCover":
os.path.splitext(issuefile)[0] + '.jpg',
"IssueDate": issuedate,
"IssueStatus": "Open"
}
myDB.upsert("magazines", newValueDict,
controlValueDict)
else:
# Set magazine_issuedate to issuedate of most recent issue we have
# Set latestcover to most recent issue cover
# Set magazine_added to acquired date of earliest issue we have
# Set magazine_lastacquired to acquired date of most recent issue we have
# acquired dates are read from magazine file timestamps
newValueDict = {"IssueStatus": "Open"}
if not magazineadded or iss_acquired < magazineadded:
newValueDict["MagazineAdded"] = iss_acquired
if not maglastacquired or iss_acquired > maglastacquired:
newValueDict["LastAcquired"] = iss_acquired
if not magissuedate or issuedate >= magissuedate:
newValueDict["IssueDate"] = issuedate
newValueDict["LatestCover"] = os.path.splitext(
issuefile)[0] + '.jpg'
myDB.upsert("magazines", newValueDict,
controlValueDict)
if lazylibrarian.CONFIG['FULL_SCAN'] and not onetitle:
magcount = myDB.match("select count(*) from magazines")
isscount = myDB.match("select count(*) from issues")
logger.info(
"Magazine scan complete, found %s magazine%s, %s issue%s" %
(magcount['count(*)'], plural(magcount['count(*)']),
isscount['count(*)'], plural(isscount['count(*)'])))
else:
logger.info("Magazine scan complete")
lazylibrarian.MAG_UPDATE = 0
except Exception:
lazylibrarian.MAG_UPDATE = 0
logger.error('Unhandled exception in magazineScan: %s' %
traceback.format_exc())
def create_id(issuename=None):
hashID = sha1(makeBytestr(issuename)).hexdigest()
# logger.debug('Issue %s Hash: %s' % (issuename, hashID))
return hashID
def magazineScan(title=None):
lazylibrarian.MAG_UPDATE = 1
# noinspection PyBroadException
try:
myDB = database.DBConnection()
onetitle = title
if onetitle:
mag_path = lazylibrarian.CONFIG['MAG_DEST_FOLDER'].replace('$Title', onetitle)
else:
mag_path = os.path.dirname(lazylibrarian.CONFIG['MAG_DEST_FOLDER'])
if lazylibrarian.CONFIG['MAG_RELATIVE']:
mag_path = os.path.join(lazylibrarian.DIRECTORY('eBook'), mag_path)
if PY2:
mag_path = mag_path.encode(lazylibrarian.SYS_ENCODING)
if lazylibrarian.CONFIG['FULL_SCAN'] and not onetitle:
mags = myDB.select('select * from Issues')
# check all the issues are still there, delete entry if not
for mag in mags:
title = mag['Title']
issuedate = mag['IssueDate']
issuefile = mag['IssueFile']
if issuefile and not os.path.isfile(issuefile):
myDB.action('DELETE from Issues where issuefile=?', (issuefile,))
logger.info('Issue %s - %s deleted as not found on disk' % (title, issuedate))
controlValueDict = {"Title": title}
newValueDict = {
"LastAcquired": None, # clear magazine dates
"IssueDate": None, # we will fill them in again later
"LatestCover": None,
"IssueStatus": "Skipped" # assume there are no issues now
}
myDB.upsert("magazines", newValueDict, controlValueDict)
logger.debug('Magazine %s details reset' % title)
# now check the magazine titles and delete any with no issues
if lazylibrarian.CONFIG['MAG_DELFOLDER']:
mags = myDB.select('SELECT Title,count(Title) as counter from issues group by Title')
for mag in mags:
title = mag['Title']
issues = mag['counter']
if not issues:
logger.debug('Magazine %s deleted as no issues found' % title)
myDB.action('DELETE from magazines WHERE Title=?', (title,))
logger.info(' Checking [%s] for magazines' % mag_path)
matchString = ''
for char in lazylibrarian.CONFIG['MAG_DEST_FILE']:
matchString = matchString + '\\' + char
# massage the MAG_DEST_FILE config parameter into something we can use
# with regular expression matching
booktypes = ''
count = -1
booktype_list = getList(lazylibrarian.CONFIG['MAG_TYPE'])
for book_type in booktype_list:
count += 1
if count == 0:
booktypes = book_type
else:
booktypes = booktypes + '|' + book_type
match = matchString.replace("\\$\\I\\s\\s\\u\\e\\D\\a\\t\\e", "(?P<issuedate>.*?)").replace(
"\\$\\T\\i\\t\\l\\e", "(?P<title>.*?)") + '\.[' + booktypes + ']'
title_pattern = re.compile(match, re.VERBOSE)
match = matchString.replace("\\$\\I\\s\\s\\u\\e\\D\\a\\t\\e", "(?P<issuedate>.*?)").replace(
"\\$\\T\\i\\t\\l\\e", "") + '\.[' + booktypes + ']'
date_pattern = re.compile(match, re.VERBOSE)
# try to ensure startdir is str as os.walk can fail if it tries to convert a subdir or file
# to utf-8 and fails (eg scandinavian characters in ascii 8bit)
for rootdir, dirnames, filenames in os.walk(makeBytestr(mag_path)):
rootdir = makeUnicode(rootdir)
filenames = [makeUnicode(item) for item in filenames]
for fname in filenames:
# maybe not all magazines will be pdf?
if is_valid_booktype(fname, booktype='mag'):
issuedate = ''
# noinspection PyBroadException
try:
match = title_pattern.match(fname)
if match:
title = match.group("title")
issuedate = match.group("issuedate")
if lazylibrarian.LOGLEVEL & lazylibrarian.log_magdates:
logger.debug("Title pattern [%s][%s]" % (title, issuedate))
match = True
else:
logger.debug("Title pattern match failed for [%s]" % fname)
except Exception:
match = False
if not match:
# noinspection PyBroadException
try:
match = date_pattern.match(fname)
if match:
issuedate = match.group("issuedate")
title = os.path.basename(rootdir)
if lazylibrarian.LOGLEVEL & lazylibrarian.log_magdates:
logger.debug("Date pattern [%s][%s]" % (title, issuedate))
match = True
else:
logger.debug("Date pattern match failed for [%s]" % fname)
except Exception:
match = False
if not match:
title = os.path.basename(rootdir)
issuedate = ''
dic = {'.': ' ', '-': ' ', '/': ' ', '+': ' ', '_': ' ', '(': '', ')': '', '[': ' ', ']': ' ',
'#': '# '}
if issuedate:
exploded = replace_all(issuedate, dic).split()
regex_pass, issuedate, year = lazylibrarian.searchmag.get_issue_date(exploded)
if lazylibrarian.LOGLEVEL & lazylibrarian.log_magdates:
logger.debug("Date regex [%s][%s][%s]" % (regex_pass, issuedate, year))
if not regex_pass:
issuedate = ''
if not issuedate:
exploded = replace_all(fname, dic).split()
regex_pass, issuedate, year = lazylibrarian.searchmag.get_issue_date(exploded)
if lazylibrarian.LOGLEVEL & lazylibrarian.log_magdates:
logger.debug("File regex [%s][%s][%s]" % (regex_pass, issuedate, year))
if not regex_pass:
issuedate = ''
if not issuedate:
logger.warn("Invalid name format for [%s]" % fname)
continue
issuefile = os.path.join(rootdir, fname) # full path to issue.pdf
mtime = os.path.getmtime(issuefile)
iss_acquired = datetime.date.isoformat(datetime.date.fromtimestamp(mtime))
if lazylibrarian.CONFIG['MAG_RENAME']:
filedate = issuedate
if issuedate and issuedate.isdigit():
if len(issuedate) == 8:
if check_year(issuedate[:4]):
filedate = 'Issue %d %s' % (int(issuedate[4:]), issuedate[:4])
else:
filedate = 'Vol %d Iss %d' % (int(issuedate[:4]), int(issuedate[4:]))
elif len(issuedate) == 12:
filedate = 'Vol %d Iss %d %s' % (int(issuedate[4:8]), int(issuedate[8:]),
issuedate[:4])
else:
filedate = str(issuedate).zfill(4)
extn = os.path.splitext(fname)[1]
newfname = lazylibrarian.CONFIG['MAG_DEST_FILE'].replace('$Title', title).replace(
'$IssueDate', filedate)
newfname = newfname + extn
if newfname and newfname != fname:
logger.debug("Rename %s -> %s" % (fname, newfname))
newissuefile = os.path.join(rootdir, newfname)
newissuefile = safe_move(issuefile, newissuefile)
if os.path.exists(issuefile.replace(extn, '.jpg')):
safe_move(issuefile.replace(extn, '.jpg'), newissuefile.replace(extn, '.jpg'))
if os.path.exists(issuefile.replace(extn, '.opf')):
safe_move(issuefile.replace(extn, '.opf'), newissuefile.replace(extn, '.opf'))
issuefile = newissuefile
logger.debug("Found %s Issue %s" % (title, issuedate))
controlValueDict = {"Title": title}
# is this magazine already in the database?
mag_entry = myDB.match(
'SELECT LastAcquired,IssueDate,MagazineAdded,CoverPage from magazines WHERE Title=?', (title,))
if not mag_entry:
# need to add a new magazine to the database
newValueDict = {
"Reject": None,
"Status": "Active",
"MagazineAdded": None,
"LastAcquired": None,
"LatestCover": None,
"IssueDate": None,
"IssueStatus": "Skipped",
"Regex": None,
"CoverPage": 1
}
logger.debug("Adding magazine %s" % title)
myDB.upsert("magazines", newValueDict, controlValueDict)
magissuedate = None
magazineadded = None
maglastacquired = None
magcoverpage = 1
else:
maglastacquired = mag_entry['LastAcquired']
magissuedate = mag_entry['IssueDate']
magazineadded = mag_entry['MagazineAdded']
magissuedate = str(magissuedate).zfill(4)
magcoverpage = mag_entry['CoverPage']
issuedate = str(issuedate).zfill(4) # for sorting issue numbers
# is this issue already in the database?
issue_id = create_id("%s %s" % (title, issuedate))
iss_entry = myDB.match('SELECT Title,IssueFile from issues WHERE Title=? and IssueDate=?',
(title, issuedate))
new_entry = False
if not iss_entry or iss_entry['IssueFile'] != issuefile:
new_entry = True # new entry or name changed
if not iss_entry:
logger.debug("Adding issue %s %s" % (title, issuedate))
else:
logger.debug("Updating issue %s %s" % (title, issuedate))
controlValueDict = {"Title": title, "IssueDate": issuedate}
newValueDict = {
"IssueAcquired": iss_acquired,
"IssueID": issue_id,
"IssueFile": issuefile
}
myDB.upsert("Issues", newValueDict, controlValueDict)
ignorefile = os.path.join(os.path.dirname(issuefile), '.ll_ignore')
with open(ignorefile, 'a'):
os.utime(ignorefile, None)
createMagCover(issuefile, pagenum=magcoverpage, refresh=new_entry)
lazylibrarian.postprocess.processMAGOPF(issuefile, title, issuedate, issue_id, overwrite=new_entry)
# see if this issues date values are useful
controlValueDict = {"Title": title}
if not mag_entry: # new magazine, this is the only issue
newValueDict = {
"MagazineAdded": iss_acquired,
"LastAcquired": iss_acquired,
"LatestCover": os.path.splitext(issuefile)[0] + '.jpg',
"IssueDate": issuedate,
"IssueStatus": "Open"
}
myDB.upsert("magazines", newValueDict, controlValueDict)
else:
# Set magazine_issuedate to issuedate of most recent issue we have
# Set latestcover to most recent issue cover
# Set magazine_added to acquired date of earliest issue we have
# Set magazine_lastacquired to acquired date of most recent issue we have
# acquired dates are read from magazine file timestamps
newValueDict = {"IssueStatus": "Open"}
if not magazineadded or iss_acquired < magazineadded:
newValueDict["MagazineAdded"] = iss_acquired
if not maglastacquired or iss_acquired > maglastacquired:
newValueDict["LastAcquired"] = iss_acquired
if not magissuedate or issuedate >= magissuedate:
newValueDict["IssueDate"] = issuedate
newValueDict["LatestCover"] = os.path.splitext(issuefile)[0] + '.jpg'
myDB.upsert("magazines", newValueDict, controlValueDict)
if lazylibrarian.CONFIG['FULL_SCAN'] and not onetitle:
magcount = myDB.match("select count(*) from magazines")
isscount = myDB.match("select count(*) from issues")
logger.info("Magazine scan complete, found %s magazine%s, %s issue%s" %
(magcount['count(*)'], plural(magcount['count(*)']),
isscount['count(*)'], plural(isscount['count(*)'])))
else:
logger.info("Magazine scan complete")
lazylibrarian.MAG_UPDATE = 0
except Exception:
lazylibrarian.MAG_UPDATE = 0
logger.error('Unhandled exception in magazineScan: %s' % traceback.format_exc())
def encode_multipart(fields, files, boundary=None):
r"""Encode dict of form fields and dict of files as multipart/form-data.
Return tuple of (body_string, headers_dict). Each value in files is a dict
with required keys 'filename' and 'content', and optional 'mimetype' (if
not specified, tries to guess mime type or uses 'application/octet-stream').
>>> body, headers = encode_multipart({'FIELD': 'VALUE'},
... {'FILE': {'filename': 'F.TXT', 'content': 'CONTENT'}},
... boundary='BOUNDARY')
>>> print('\n'.join(repr(l) for l in body.split('\r\n')))
'--BOUNDARY'
'Content-Disposition: form-data; name="FIELD"'
''
'VALUE'
'--BOUNDARY'
'Content-Disposition: form-data; name="FILE"; filename="F.TXT"'
'Content-Type: text/plain'
''
'CONTENT'
'--BOUNDARY--'
''
>>> print(sorted(headers.items()))
[('Content-Length', '193'), ('Content-Type', 'multipart/form-data; boundary=BOUNDARY')]
>>> len(body)
193
"""
def escape_quote(s):
s = makeUnicode(s)
return s.replace('"', '\\"')
if boundary is None:
boundary = ''.join(random.choice(_BOUNDARY_CHARS) for _ in range(30))
lines = []
if fields:
fields = dict((makeBytestr(k), makeBytestr(v)) for k, v in fields.items())
for name, value in list(fields.items()):
lines.extend((
'--{0}'.format(boundary),
'Content-Disposition: form-data; name="{0}"'.format(escape_quote(name)),
'',
makeUnicode(value),
))
if files:
for name, value in list(files.items()):
filename = value['filename']
if 'mimetype' in value:
mimetype = value['mimetype']
else:
mimetype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
lines.extend((
'--{0}'.format(boundary),
'Content-Disposition: form-data; name="{0}"; filename="{1}"'.format(
escape_quote(name), escape_quote(filename)),
'Content-Type: {0}'.format(mimetype),
'',
value['content'],
))
lines.extend((
'--{0}--'.format(boundary),
'',
))
body = makeBytestr('\r\n'.join(lines))
headers = {
'Content-Type': 'multipart/form-data; boundary={0}'.format(boundary),
'Content-Length': str(len(body)),
}
return body, headers
def magazineScan():
lazylibrarian.MAG_UPDATE = 1
# noinspection PyBroadException
try:
myDB = database.DBConnection()
mag_path = lazylibrarian.CONFIG['MAG_DEST_FOLDER']
mag_path = mag_path.split('$')[0]
if lazylibrarian.CONFIG['MAG_RELATIVE']:
if mag_path[0] not in '._':
mag_path = '_' + mag_path
mag_path = os.path.join(lazylibrarian.DIRECTORY('eBook'), mag_path)
mag_path = mag_path.encode(lazylibrarian.SYS_ENCODING)
if lazylibrarian.CONFIG['FULL_SCAN']:
mags = myDB.select('select * from Issues')
# check all the issues are still there, delete entry if not
for mag in mags:
title = mag['Title']
issuedate = mag['IssueDate']
issuefile = mag['IssueFile']
if issuefile and not os.path.isfile(issuefile):
myDB.action('DELETE from Issues where issuefile=?',
(issuefile, ))
logger.info('Issue %s - %s deleted as not found on disk' %
(title, issuedate))
controlValueDict = {"Title": title}
newValueDict = {
"LastAcquired": None, # clear magazine dates
"IssueDate": None, # we will fill them in again later
"LatestCover": None,
"IssueStatus":
"Skipped" # assume there are no issues now
}
myDB.upsert("magazines", newValueDict, controlValueDict)
logger.debug('Magazine %s details reset' % title)
mags = myDB.select('SELECT * from magazines')
# now check the magazine titles and delete any with no issues
for mag in mags:
title = mag['Title']
count = myDB.select(
'SELECT COUNT(Title) as counter FROM issues WHERE Title=?',
(title, ))
issues = count[0]['counter']
if not issues:
logger.debug('Magazine %s deleted as no issues found' %
title)
myDB.action('DELETE from magazines WHERE Title=?',
(title, ))
logger.info(' Checking [%s] for magazines' % mag_path)
matchString = ''
for char in lazylibrarian.CONFIG['MAG_DEST_FILE']:
matchString = matchString + '\\' + char
# massage the MAG_DEST_FILE config parameter into something we can use
# with regular expression matching
booktypes = ''
count = -1
booktype_list = getList(lazylibrarian.CONFIG['MAG_TYPE'])
for book_type in booktype_list:
count += 1
if count == 0:
booktypes = book_type
else:
booktypes = booktypes + '|' + book_type
match = matchString.replace(
"\\$\\I\\s\\s\\u\\e\\D\\a\\t\\e", "(?P<issuedate>.*?)").replace(
"\\$\\T\\i\\t\\l\\e",
"(?P<title>.*?)") + '\.[' + booktypes + ']'
title_pattern = re.compile(match, re.VERBOSE)
match = matchString.replace(
"\\$\\I\\s\\s\\u\\e\\D\\a\\t\\e", "(?P<issuedate>.*?)").replace(
"\\$\\T\\i\\t\\l\\e", "") + '\.[' + booktypes + ']'
date_pattern = re.compile(match, re.VERBOSE)
# try to ensure startdir is str as os.walk can fail if it tries to convert a subdir or file
# to utf-8 and fails (eg scandinavian characters in ascii 8bit)
for rootdir, dirnames, filenames in os.walk(makeBytestr(mag_path)):
rootdir = makeUnicode(rootdir)
filenames = [makeUnicode(item) for item in filenames]
for fname in filenames:
# maybe not all magazines will be pdf?
if is_valid_booktype(fname, booktype='mag'):
issuedate = ''
# noinspection PyBroadException
try:
match = title_pattern.match(fname)
if match:
issuedate = match.group("issuedate")
title = match.group("title")
match = True
else:
match = False
except Exception:
match = False
if not match:
try:
match = date_pattern.match(fname)
if match:
issuedate = match.group("issuedate")
title = os.path.basename(rootdir)
else:
logger.debug("Pattern match failed for [%s]" %
fname)
continue
except Exception as e:
logger.debug("Invalid name format for [%s] %s %s" %
(fname, type(e).__name__, str(e)))
continue
logger.debug("Found %s Issue %s" % (title, fname))
issuefile = os.path.join(rootdir,
fname) # full path to issue.pdf
mtime = os.path.getmtime(issuefile)
iss_acquired = datetime.date.isoformat(
datetime.date.fromtimestamp(mtime))
controlValueDict = {"Title": title}
# is this magazine already in the database?
mag_entry = myDB.match(
'SELECT LastAcquired, IssueDate, MagazineAdded from magazines WHERE Title=?',
(title, ))
if not mag_entry:
# need to add a new magazine to the database
newValueDict = {
"Reject": None,
"Status": "Active",
"MagazineAdded": None,
"LastAcquired": None,
"LatestCover": None,
"IssueDate": None,
"IssueStatus": "Skipped",
"Regex": None
}
logger.debug("Adding magazine %s" % title)
myDB.upsert("magazines", newValueDict,
controlValueDict)
magissuedate = None
magazineadded = None
else:
maglastacquired = mag_entry['LastAcquired']
magissuedate = mag_entry['IssueDate']
magazineadded = mag_entry['MagazineAdded']
magissuedate = str(magissuedate).zfill(4)
issuedate = str(issuedate).zfill(
4) # for sorting issue numbers
# is this issue already in the database?
controlValueDict = {"Title": title, "IssueDate": issuedate}
issue_id = create_id("%s %s" % (title, issuedate))
iss_entry = myDB.match(
'SELECT Title from issues WHERE Title=? and IssueDate=?',
(title, issuedate))
if not iss_entry:
newValueDict = {
"IssueAcquired": iss_acquired,
"IssueID": issue_id,
"IssueFile": issuefile
}
myDB.upsert("Issues", newValueDict, controlValueDict)
logger.debug("Adding issue %s %s" % (title, issuedate))
create_cover(issuefile)
lazylibrarian.postprocess.processMAGOPF(
issuefile, title, issuedate, issue_id)
# see if this issues date values are useful
controlValueDict = {"Title": title}
if not mag_entry: # new magazine, this is the only issue
newValueDict = {
"MagazineAdded": iss_acquired,
"LastAcquired": iss_acquired,
"LatestCover":
os.path.splitext(issuefile)[0] + '.jpg',
"IssueDate": issuedate,
"IssueStatus": "Open"
}
myDB.upsert("magazines", newValueDict,
controlValueDict)
else:
# Set magazine_issuedate to issuedate of most recent issue we have
# Set latestcover to most recent issue cover
# Set magazine_added to acquired date of earliest issue we have
# Set magazine_lastacquired to acquired date of most recent issue we have
# acquired dates are read from magazine file timestamps
newValueDict = {"IssueStatus": "Open"}
if not magazineadded or iss_acquired < magazineadded:
newValueDict["MagazineAdded"] = iss_acquired
if not maglastacquired or iss_acquired > maglastacquired:
newValueDict["LastAcquired"] = iss_acquired
if not magissuedate or issuedate >= magissuedate:
newValueDict["IssueDate"] = issuedate
newValueDict["LatestCover"] = os.path.splitext(
issuefile)[0] + '.jpg'
myDB.upsert("magazines", newValueDict,
controlValueDict)
magcount = myDB.match("select count(*) from magazines")
isscount = myDB.match("select count(*) from issues")
logger.info("Magazine scan complete, found %s magazine%s, %s issue%s" %
(magcount['count(*)'], plural(magcount['count(*)']),
isscount['count(*)'], plural(isscount['count(*)'])))
lazylibrarian.MAG_UPDATE = 0
except Exception:
lazylibrarian.MAG_UPDATE = 0
logger.error('Unhandled exception in magazineScan: %s' %
traceback.format_exc())
def audioRename(bookid):
for item in ['$Part', '$Title']:
if item not in lazylibrarian.CONFIG['AUDIOBOOK_DEST_FILE']:
logger.error("Unable to audioRename, check AUDIOBOOK_DEST_FILE")
return ''
myDB = database.DBConnection()
cmd = 'select AuthorName,BookName,AudioFile from books,authors where books.AuthorID = authors.AuthorID and bookid=?'
exists = myDB.match(cmd, (bookid,))
if exists:
book_filename = exists['AudioFile']
if book_filename:
r = os.path.dirname(book_filename)
else:
logger.debug("No filename for %s in audioRename %s" % bookid)
return ''
else:
logger.debug("Invalid bookid in audioRename %s" % bookid)
return ''
if not TinyTag:
logger.warn("TinyTag library not available")
return ''
cnt = 0
parts = []
author = ''
book = ''
total = 0
audio_file = ''
for f in os.listdir(makeBytestr(r)):
f = makeUnicode(f)
if is_valid_booktype(f, booktype='audiobook'):
cnt += 1
audio_file = f
try:
id3r = TinyTag.get(os.path.join(r, f))
performer = id3r.artist
composer = id3r.composer
book = id3r.album
track = id3r.track
total = id3r.track_total
track = check_int(track, 0)
total = check_int(total, 0)
if composer: # if present, should be author
author = composer
elif performer: # author, or narrator if composer == author
author = performer
if author and book:
parts.append([track, book, author, f])
except Exception as e:
logger.error("tinytag %s %s" % (type(e).__name__, str(e)))
pass
logger.debug("%s found %s audiofile%s" % (exists['BookName'], cnt, plural(cnt)))
if cnt == 1 and not parts: # single file audiobook
parts = [1, exists['BookName'], exists['AuthorName'], audio_file]
if cnt != len(parts):
logger.warn("%s: Incorrect number of parts (found %i from %i)" % (exists['BookName'], len(parts), cnt))
return book_filename
if total and total != cnt:
logger.warn("%s: Reported %i parts, got %i" % (exists['BookName'], total, cnt))
return book_filename
# check all parts have the same author and title
if len(parts) > 1:
for part in parts:
if part[1] != book:
logger.warn("%s: Inconsistent title: [%s][%s]" % (exists['BookName'], part[1], book))
return book_filename
if part[2] != author:
logger.warn("%s: Inconsistent author: [%s][%s]" % (exists['BookName'], part[2], author))
return book_filename
# do we have any track info (value is 0 if not)
if parts[0][0] == 0:
tokmatch = ''
# try to extract part information from filename. Search for token style of part 1 in this order...
for token in [' 001.', ' 01.', ' 1.', ' 001 ', ' 01 ', ' 1 ', '01']:
if tokmatch:
break
for part in parts:
if token in part[3]:
tokmatch = token
break
if tokmatch: # we know the numbering style, get numbers for the other parts
cnt = 0
while cnt < len(parts):
cnt += 1
if tokmatch == ' 001.':
pattern = ' %s.' % str(cnt).zfill(3)
elif tokmatch == ' 01.':
pattern = ' %s.' % str(cnt).zfill(2)
elif tokmatch == ' 1.':
pattern = ' %s.' % str(cnt)
elif tokmatch == ' 001 ':
pattern = ' %s ' % str(cnt).zfill(3)
elif tokmatch == ' 01 ':
pattern = ' %s ' % str(cnt).zfill(2)
elif tokmatch == ' 1 ':
pattern = ' %s ' % str(cnt)
else:
pattern = '%s' % str(cnt).zfill(2)
# standardise numbering of the parts
for part in parts:
if pattern in part[3]:
part[0] = cnt
break
# check all parts are present
cnt = 0
found = True
while found and cnt < len(parts):
found = False
cnt += 1
for part in parts:
trk = part[0]
if trk == cnt:
found = True
break
if not found:
logger.warn("%s: No part %i found" % (exists['BookName'], cnt))
return book_filename
# if we get here, looks like we have all the parts needed to rename properly
seriesinfo = seriesInfo(bookid)
dest_path = lazylibrarian.CONFIG['EBOOK_DEST_FOLDER'].replace(
'$Author', author).replace(
'$Title', book).replace(
'$Series', seriesinfo['Full']).replace(
'$SerName', seriesinfo['Name']).replace(
'$SerNum', seriesinfo['Num']).replace(
'$$', ' ')
dest_path = ' '.join(dest_path.split()).strip()
dest_path = replace_all(dest_path, __dic__)
dest_dir = lazylibrarian.DIRECTORY('Audio')
dest_path = os.path.join(dest_dir, dest_path)
if r != dest_path:
try:
dest_path = safe_move(r, dest_path)
r = dest_path
except Exception as why:
if not os.path.isdir(dest_path):
logger.error('Unable to create directory %s: %s' % (dest_path, why))
for part in parts:
pattern = lazylibrarian.CONFIG['AUDIOBOOK_DEST_FILE']
seriesinfo = seriesInfo(bookid)
pattern = pattern.replace(
'$Author', author).replace(
'$Title', book).replace(
'$Part', str(part[0]).zfill(len(str(len(parts))))).replace(
'$Total', str(len(parts))).replace(
'$Series', seriesinfo['Full']).replace(
'$SerName', seriesinfo['Name']).replace(
'$SerNum', seriesinfo['Num']).replace(
'$$', ' ')
pattern = ' '.join(pattern.split()).strip()
n = os.path.join(r, pattern + os.path.splitext(part[3])[1])
o = os.path.join(r, part[3])
if o != n:
try:
n = safe_move(o, n)
if part[0] == 1:
book_filename = n # return part 1 of set
logger.debug('%s: audioRename [%s] to [%s]' % (exists['BookName'], o, n))
except Exception as e:
logger.error('Unable to rename [%s] to [%s] %s %s' % (o, n, type(e).__name__, str(e)))
return book_filename
def get_cached_request(url, useCache=True, cache="XML"):
# hashfilename = hash of url
# if hashfilename exists in cache and isn't too old, return its contents
# if not, read url and store the result in the cache
# return the result, and boolean True if source was cache
#
cacheLocation = cache + "Cache"
cacheLocation = os.path.join(lazylibrarian.CACHEDIR, cacheLocation)
if not os.path.exists(cacheLocation):
os.mkdir(cacheLocation)
myhash = md5_utf8(url)
valid_cache = False
source = None
hashfilename = cacheLocation + os.path.sep + myhash + "." + cache.lower()
expiry = lazylibrarian.CONFIG['CACHE_AGE'] * 24 * 60 * 60 # expire cache after this many seconds
if useCache and os.path.isfile(hashfilename):
cache_modified_time = os.stat(hashfilename).st_mtime
time_now = time.time()
if cache_modified_time < time_now - expiry:
# Cache entry is too old, delete it
if lazylibrarian.LOGLEVEL & lazylibrarian.log_cache:
logger.debug("Expiring %s" % myhash)
os.remove(hashfilename)
else:
valid_cache = True
if valid_cache:
lazylibrarian.CACHE_HIT = int(lazylibrarian.CACHE_HIT) + 1
if lazylibrarian.LOGLEVEL & lazylibrarian.log_cache:
logger.debug("CacheHandler: Returning CACHED response %s for %s" % (hashfilename, url))
if cache == "JSON":
try:
source = json.load(open(hashfilename))
except ValueError:
logger.error("Error decoding json from %s" % hashfilename)
return None, False
elif cache == "XML":
with open(hashfilename, "rb") as cachefile:
result = cachefile.read()
if result and result.startswith(b'<?xml'):
try:
source = ElementTree.fromstring(result)
except UnicodeEncodeError:
# seems sometimes the page contains utf-16 but the header says it's utf-8
try:
result = result.decode('utf-16').encode('utf-8')
source = ElementTree.fromstring(result)
except (ElementTree.ParseError, UnicodeEncodeError, UnicodeDecodeError):
logger.error("Error parsing xml from %s" % hashfilename)
source = None
except ElementTree.ParseError:
logger.error("Error parsing xml from %s" % hashfilename)
source = None
if source is None:
logger.error("Error reading xml from %s" % hashfilename)
os.remove(hashfilename)
return None, False
else:
lazylibrarian.CACHE_MISS = int(lazylibrarian.CACHE_MISS) + 1
if cache == 'XML':
gr_api_sleep()
result, success = fetchURL(url, raw=True)
else:
result, success = fetchURL(url)
if success:
if lazylibrarian.LOGLEVEL & lazylibrarian.log_cache:
logger.debug("CacheHandler: Storing %s %s for %s" % (cache, myhash, url))
if cache == "JSON":
try:
source = json.loads(result)
if not expiry:
return source, False
except Exception as e:
logger.error("%s decoding json from %s" % (type(e).__name__, url))
logger.debug("%s : %s" % (e, result))
return None, False
json.dump(source, open(hashfilename, "w"))
elif cache == "XML":
result = makeBytestr(result)
if result and result.startswith(b'<?xml'):
try:
source = ElementTree.fromstring(result)
if not expiry:
return source, False
except UnicodeEncodeError:
# sometimes we get utf-16 data labelled as utf-8
try:
result = result.decode('utf-16').encode('utf-8')
source = ElementTree.fromstring(result)
if not expiry:
return source, False
except (ElementTree.ParseError, UnicodeEncodeError, UnicodeDecodeError):
logger.error("Error parsing xml from %s" % url)
source = None
except ElementTree.ParseError:
logger.error("Error parsing xml from %s" % url)
source = None
if source is not None:
with open(hashfilename, "wb") as cachefile:
cachefile.write(result)
else:
logger.error("Error getting xml data from %s" % url)
return None, False
else:
logger.debug("Got error response for %s: %s" % (url, result.split('<')[0]))
if 'goodreads' in url and '503' in result:
time.sleep(1)
return None, False
return source, valid_cache
def createMagCover(issuefile=None, refresh=False):
if not lazylibrarian.CONFIG['IMP_MAGCOVER']:
return
if issuefile is None or not os.path.isfile(issuefile):
logger.debug('No issuefile %s' % issuefile)
return
base, extn = os.path.splitext(issuefile)
if not extn:
logger.debug('Unable to create cover for %s, no extension?' % issuefile)
return
coverfile = base + '.jpg'
if os.path.isfile(coverfile):
if refresh:
os.remove(coverfile)
else:
logger.debug('Cover for %s exists' % issuefile)
return # quit if cover already exists and we didn't want to refresh
logger.debug('Creating cover for %s' % issuefile)
data = '' # result from unzip or unrar
extn = extn.lower()
if extn in ['.cbz', '.epub']:
try:
data = zipfile.ZipFile(issuefile)
except Exception as why:
logger.error("Failed to read zip file %s, %s %s" % (issuefile, type(why).__name__, str(why)))
data = ''
elif extn in ['.cbr']:
try:
# unrar will complain if the library isn't installed, needs to be compiled separately
# see https://pypi.python.org/pypi/unrar/ for instructions
# Download source from http://www.rarlab.com/rar_add.htm
# note we need LIBRARY SOURCE not a binary package
# make lib; sudo make install-lib; sudo ldconfig
# lib.unrar should then be able to find libunrar.so
from lib.unrar import rarfile
data = rarfile.RarFile(issuefile)
except Exception as why:
logger.error("Failed to read rar file %s, %s %s" % (issuefile, type(why).__name__, str(why)))
data = ''
if data:
img = None
try:
for member in data.namelist():
memlow = member.lower()
if '-00.' in memlow or '000.' in memlow or 'cover.' in memlow:
if memlow.endswith('.jpg') or memlow.endswith('.jpeg'):
img = data.read(member)
break
if img:
with open(coverfile, 'wb') as f:
if PY2:
f.write(img)
else:
f.write(img.encode())
return
else:
logger.debug("Failed to find image in %s" % issuefile)
except Exception as why:
logger.error("Failed to extract image from %s, %s %s" % (issuefile, type(why).__name__, str(why)))
elif extn == '.pdf':
generator = ""
if len(lazylibrarian.CONFIG['IMP_CONVERT']): # allow external convert to override libraries
generator = "external program: %s" % lazylibrarian.CONFIG['IMP_CONVERT']
if "gsconvert.py" in lazylibrarian.CONFIG['IMP_CONVERT']:
msg = "Use of gsconvert.py is deprecated, equivalent functionality is now built in. "
msg += "Support for gsconvert.py may be removed in a future release. See wiki for details."
logger.warn(msg)
converter = lazylibrarian.CONFIG['IMP_CONVERT']
postfix = ''
# if not os.path.isfile(converter): # full path given, or just program_name?
# converter = os.path.join(os.getcwd(), lazylibrarian.CONFIG['IMP_CONVERT'])
if 'convert' in converter and 'gs' not in converter:
# tell imagemagick to only convert first page
postfix = '[0]'
try:
params = [converter, '%s%s' % (issuefile, postfix), '%s' % coverfile]
res = subprocess.check_output(params, stderr=subprocess.STDOUT)
res = makeUnicode(res).strip()
if res:
logger.debug('%s reports: %s' % (lazylibrarian.CONFIG['IMP_CONVERT'], res))
except Exception as e:
# logger.debug(params)
logger.warn('External "convert" failed %s %s' % (type(e).__name__, str(e)))
elif platform.system() == "Windows":
GS = os.path.join(os.getcwd(), "gswin64c.exe")
generator = "local gswin64c"
if not os.path.isfile(GS):
GS = os.path.join(os.getcwd(), "gswin32c.exe")
generator = "local gswin32c"
if not os.path.isfile(GS):
params = ["where", "gswin64c"]
try:
GS = subprocess.check_output(params, stderr=subprocess.STDOUT)
GS = makeUnicode(GS).strip()
generator = "gswin64c"
except Exception as e:
logger.debug("where gswin64c failed: %s %s" % (type(e).__name__, str(e)))
if not os.path.isfile(GS):
params = ["where", "gswin32c"]
try:
GS = subprocess.check_output(params, stderr=subprocess.STDOUT)
GS = makeUnicode(GS).strip()
generator = "gswin32c"
except Exception as e:
logger.debug("where gswin32c failed: %s %s" % (type(e).__name__, str(e)))
if not os.path.isfile(GS):
logger.debug("No gswin found")
generator = "(no windows ghostscript found)"
else:
# noinspection PyBroadException
try:
params = [GS, "--version"]
res = subprocess.check_output(params, stderr=subprocess.STDOUT)
res = makeUnicode(res).strip()
logger.debug("Found %s [%s] version %s" % (generator, GS, res))
generator = "%s version %s" % (generator, res)
issuefile = issuefile.split('[')[0]
params = [GS, "-sDEVICE=jpeg", "-dNOPAUSE", "-dBATCH", "-dSAFER", "-dFirstPage=1", "-dLastPage=1",
"-dUseCropBox", "-sOutputFile=%s" % coverfile, issuefile]
res = subprocess.check_output(params, stderr=subprocess.STDOUT)
res = makeUnicode(res).strip()
if not os.path.isfile(coverfile):
logger.debug("Failed to create jpg: %s" % res)
except Exception: # as why:
logger.warn("Failed to create jpg for %s" % issuefile)
logger.debug('Exception in gswin create_cover: %s' % traceback.format_exc())
else: # not windows
try:
# noinspection PyUnresolvedReferences
from wand.image import Image
interface = "wand"
except ImportError:
try:
# No PythonMagick in python3
# noinspection PyUnresolvedReferences
import PythonMagick
interface = "pythonmagick"
except ImportError:
interface = ""
try:
if interface == 'wand':
generator = "wand interface"
with Image(filename=issuefile + '[0]') as img:
img.save(filename=coverfile)
elif interface == 'pythonmagick':
generator = "pythonmagick interface"
img = PythonMagick.Image()
# PythonMagick requires filenames to be bytestr, not unicode
if type(issuefile) is text_type:
issuefile = makeBytestr(issuefile)
if type(coverfile) is text_type:
coverfile = makeBytestr(coverfile)
img.read(issuefile + '[0]')
img.write(coverfile)
else:
GS = os.path.join(os.getcwd(), "gs")
generator = "local gs"
if not os.path.isfile(GS):
GS = ""
params = ["which", "gs"]
try:
GS = subprocess.check_output(params, stderr=subprocess.STDOUT)
GS = makeUnicode(GS).strip()
generator = GS
except Exception as e:
logger.debug("which gs failed: %s %s" % (type(e).__name__, str(e)))
if not os.path.isfile(GS):
logger.debug("Cannot find gs")
generator = "(no gs found)"
else:
params = [GS, "--version"]
res = subprocess.check_output(params, stderr=subprocess.STDOUT)
res = makeUnicode(res).strip()
logger.debug("Found gs [%s] version %s" % (GS, res))
generator = "%s version %s" % (generator, res)
issuefile = issuefile.split('[')[0]
params = [GS, "-sDEVICE=jpeg", "-dNOPAUSE", "-dBATCH", "-dSAFER", "-dFirstPage=1",
"-dLastPage=1", "-dUseCropBox", "-sOutputFile=%s" % coverfile, issuefile]
res = subprocess.check_output(params, stderr=subprocess.STDOUT)
res = makeUnicode(res).strip()
if not os.path.isfile(coverfile):
logger.debug("Failed to create jpg: %s" % res)
except Exception as e:
logger.warn("Unable to create cover for %s using %s %s" % (issuefile, type(e).__name__, generator))
logger.debug('Exception in create_cover: %s' % traceback.format_exc())
if os.path.isfile(coverfile):
setperm(coverfile)
logger.debug("Created cover for %s using %s" % (issuefile, generator))
return
# if not recognised extension or cover creation failed
try:
coverfile = safe_copy(os.path.join(lazylibrarian.PROG_DIR, 'data/images/nocover.jpg'), coverfile)
setperm(coverfile)
except Exception as why:
logger.error("Failed to copy nocover file, %s %s" % (type(why).__name__, str(why)))
return
def get_cached_request(url, useCache=True, cache="XML"):
# hashfilename = hash of url
# if hashfilename exists in cache and isn't too old, return its contents
# if not, read url and store the result in the cache
# return the result, and boolean True if source was cache
#
cacheLocation = cache + "Cache"
cacheLocation = os.path.join(lazylibrarian.CACHEDIR, cacheLocation)
if not os.path.exists(cacheLocation):
os.mkdir(cacheLocation)
myhash = md5_utf8(url)
valid_cache = False
source = None
hashfilename = cacheLocation + os.path.sep + myhash + "." + cache.lower()
expiry = lazylibrarian.CONFIG['CACHE_AGE'] * 24 * 60 * 60 # expire cache after this many seconds
if useCache and os.path.isfile(hashfilename):
cache_modified_time = os.stat(hashfilename).st_mtime
time_now = time.time()
if cache_modified_time < time_now - expiry:
# Cache entry is too old, delete it
if lazylibrarian.LOGLEVEL & lazylibrarian.log_cache:
logger.debug("Expiring %s" % myhash)
os.remove(hashfilename)
else:
valid_cache = True
if valid_cache:
lazylibrarian.CACHE_HIT = int(lazylibrarian.CACHE_HIT) + 1
if lazylibrarian.LOGLEVEL & lazylibrarian.log_cache:
logger.debug("CacheHandler: Returning CACHED response %s for %s" % (hashfilename, url))
if cache == "JSON":
try:
source = json.load(open(hashfilename))
except ValueError:
logger.error("Error decoding json from %s" % hashfilename)
return None, False
elif cache == "XML":
with open(hashfilename, "rb") as cachefile:
result = cachefile.read()
if result and result.startswith(b'<?xml'):
try:
source = ElementTree.fromstring(result)
except UnicodeEncodeError:
# seems sometimes the page contains utf-16 but the header says it's utf-8
try:
result = result.decode('utf-16').encode('utf-8')
source = ElementTree.fromstring(result)
except (ElementTree.ParseError, UnicodeEncodeError, UnicodeDecodeError):
logger.error("Error parsing xml from %s" % hashfilename)
source = None
except ElementTree.ParseError:
logger.error("Error parsing xml from %s" % hashfilename)
source = None
if source is None:
logger.error("Error reading xml from %s" % hashfilename)
os.remove(hashfilename)
return None, False
else:
lazylibrarian.CACHE_MISS = int(lazylibrarian.CACHE_MISS) + 1
if cache == 'XML':
gr_api_sleep()
result, success = fetchURL(url, raw=True)
else:
result, success = fetchURL(url)
if success:
if lazylibrarian.LOGLEVEL & lazylibrarian.log_cache:
logger.debug("CacheHandler: Storing %s %s for %s" % (cache, myhash, url))
if cache == "JSON":
try:
source = json.loads(result)
if not expiry:
return source, False
except Exception as e:
logger.error("%s decoding json from %s" % (type(e).__name__, url))
logger.debug("%s : %s" % (e, result))
return None, False
json.dump(source, open(hashfilename, "w"))
elif cache == "XML":
result = makeBytestr(result)
if result and result.startswith(b'<?xml'):
try:
source = ElementTree.fromstring(result)
if not expiry:
return source, False
except UnicodeEncodeError:
# sometimes we get utf-16 data labelled as utf-8
try:
result = result.decode('utf-16').encode('utf-8')
source = ElementTree.fromstring(result)
if not expiry:
return source, False
except (ElementTree.ParseError, UnicodeEncodeError, UnicodeDecodeError):
logger.error("Error parsing xml from %s" % url)
source = None
except ElementTree.ParseError:
logger.error("Error parsing xml from %s" % url)
source = None
if source is not None:
with open(hashfilename, "wb") as cachefile:
cachefile.write(result)
else:
logger.error("Error getting xml data from %s" % url)
return None, False
else:
logger.debug("Got error response for %s: %s" % (url, result.split('<')[0]))
if 'goodreads' in url and '503' in result:
time.sleep(1)
return None, False
return source, valid_cache
