def encode_multipart(fields, files, boundary=None): """Encode dict of form fields and dict of files as multipart/form-data. Return tuple of (body_string, headers_dict). Each value in files is a dict with required keys 'filename' and 'content', and optional 'mimetype' (if not specified, tries to guess mime type or uses 'application/octet-stream'). """ def escape_quote(s): s = makeUnicode(s) return s.replace('"', '\\"') if boundary is None: boundary = ''.join(random.choice(_BOUNDARY_CHARS) for _ in range(30)) lines = [] if fields: fields = dict( (makeBytestr(k), makeBytestr(v)) for k, v in fields.items()) for name, value in list(fields.items()): lines.extend(( '--{0}'.format(boundary), 'Content-Disposition: form-data; name="{0}"'.format( escape_quote(name)), '', makeUnicode(value), )) if files: for name, value in list(files.items()): filename = value['filename'] if 'mimetype' in value: mimetype = value['mimetype'] else: mimetype = mimetypes.guess_type( filename)[0] or 'application/octet-stream' lines.extend(( '--{0}'.format(boundary), 'Content-Disposition: form-data; name="{0}"; filename="{1}"'. format(escape_quote(name), escape_quote(filename)), 'Content-Type: {0}'.format(mimetype), '', value['content'], )) lines.extend(( '--{0}--'.format(boundary), '', )) body = makeBytestr('\r\n'.join(lines)) headers = { 'Content-Type': 'multipart/form-data; boundary={0}'.format(boundary), 'Content-Length': str(len(body)), } return body, headers
def encode_multipart(fields, files, boundary=None): """Encode dict of form fields and dict of files as multipart/form-data. Return tuple of (body_string, headers_dict). Each value in files is a dict with required keys 'filename' and 'content', and optional 'mimetype' (if not specified, tries to guess mime type or uses 'application/octet-stream'). """ def escape_quote(s): s = makeUnicode(s) return s.replace('"', '\\"') if boundary is None: boundary = ''.join(random.choice(_BOUNDARY_CHARS) for _ in range(30)) lines = [] if fields: fields = dict((makeBytestr(k), makeBytestr(v)) for k, v in fields.items()) for name, value in list(fields.items()): lines.extend(( '--{0}'.format(boundary), 'Content-Disposition: form-data; name="{0}"'.format(escape_quote(name)), '', makeUnicode(value), )) if files: for name, value in list(files.items()): filename = value['filename'] if 'mimetype' in value: mimetype = value['mimetype'] else: mimetype = mimetypes.guess_type(filename)[0] or 'application/octet-stream' lines.extend(( '--{0}'.format(boundary), 'Content-Disposition: form-data; name="{0}"; filename="{1}"'.format( escape_quote(name), escape_quote(filename)), 'Content-Type: {0}'.format(mimetype), '', value['content'], )) lines.extend(( '--{0}--'.format(boundary), '', )) body = makeBytestr('\r\n'.join(lines)) headers = { 'Content-Type': 'multipart/form-data; boundary={0}'.format(boundary), 'Content-Length': str(len(body)), } return body, headers
def dump_table(table, savedir=None, status=None): myDB = database.DBConnection() # noinspection PyBroadException try: columns = myDB.select('PRAGMA table_info(%s)' % table) if not columns: # no such table logger.warn("No such table [%s]" % table) return 0 if not os.path.isdir(savedir): savedir = lazylibrarian.DATADIR headers = '' for item in columns: if headers: headers += ',' headers += item[1] if status: cmd = 'SELECT %s from %s WHERE status="%s"' % (headers, table, status) else: cmd = 'SELECT %s from %s' % (headers, table) data = myDB.select(cmd) count = 0 if data is not None: label = table if status: label += '_%s' % status csvFile = os.path.join(savedir, "%s.csv" % label) if PY2: fmode = 'wb' else: fmode = 'w' with open(csvFile, fmode) as csvfile: csvwrite = writer(csvfile, delimiter=',', quotechar='"', quoting=QUOTE_MINIMAL) headers = headers.split(',') csvwrite.writerow(headers) for item in data: if PY2: csvwrite.writerow( [makeBytestr(s) if s else '' for s in item]) else: csvwrite.writerow([str(s) if s else '' for s in item]) count += 1 msg = "Exported %s item%s to %s" % (count, plural(count), csvFile) logger.info(msg) return count except Exception: msg = 'Unhandled exception in dump_table: %s' % traceback.format_exc() logger.error(msg) return 0
def any_file(search_dir=None, extn=None): # find a file with specified extension in a directory, any will do # return full pathname of file, or empty string if none found if search_dir is None or extn is None: return "" if os.path.isdir(search_dir): for fname in os.listdir(makeBytestr(search_dir)): fname = makeUnicode(fname) if fname.endswith(extn): return os.path.join(search_dir, fname) return ""
def _get_sid(self, base_url, username, password): # login so we can capture SID cookie login_data = makeBytestr(urlencode({'username': username, 'password': password})) try: _ = self.opener.open(base_url + '/login', login_data) except Exception as err: logger.error('Error getting SID. qBittorrent %s: %s' % (type(err).__name__, str(err))) logger.warn('Unable to log in to %s/login' % base_url) return for cookie in self.cookiejar: logger.debug('login cookie: ' + cookie.name + ', value: ' + cookie.value) return
def dump_table(table, savedir=None, status=None): myDB = database.DBConnection() # noinspection PyBroadException try: columns = myDB.select('PRAGMA table_info(%s)' % table) if not columns: # no such table logger.warn("No such table [%s]" % table) return 0 if not os.path.isdir(savedir): savedir = lazylibrarian.DATADIR headers = '' for item in columns: if headers: headers += ',' headers += item[1] if status: cmd = 'SELECT %s from %s WHERE status="%s"' % (headers, table, status) else: cmd = 'SELECT %s from %s' % (headers, table) data = myDB.select(cmd) count = 0 if data is not None: label = table if status: label += '_%s' % status csvFile = os.path.join(savedir, "%s.csv" % label) if PY2: fmode = 'wb' else: fmode = 'w' with open(csvFile, fmode) as csvfile: csvwrite = writer(csvfile, delimiter=',', quotechar='"', quoting=QUOTE_MINIMAL) headers = headers.split(',') csvwrite.writerow(headers) for item in data: if PY2: csvwrite.writerow([makeBytestr(s) if s else '' for s in item]) else: csvwrite.writerow([str(s) if s else '' for s in item]) count += 1 msg = "Exported %s item%s to %s" % (count, plural(count), csvFile) logger.info(msg) return count except Exception: msg = 'Unhandled exception in dump_table: %s' % traceback.format_exc() logger.error(msg) return 0
def _command(self, command, args=None, content_type=None, files=None): logger.debug('QBittorrent WebAPI Command: %s' % command) url = self.base_url + '/' + command data = None headers = dict() if files or content_type == 'multipart/form-data': data, headers = encode_multipart( args, files, '-------------------------acebdf13572468') else: if args: data = makeBytestr(urlencode(args)) if content_type: headers['Content-Type'] = content_type request = Request(url, data, headers) if lazylibrarian.CONFIG['PROXY_HOST']: for item in getList(lazylibrarian.CONFIG['PROXY_TYPE']): request.set_proxy(lazylibrarian.CONFIG['PROXY_HOST'], item) request.add_header('User-Agent', USER_AGENT) try: response = self.opener.open(request) try: contentType = response.headers['content-type'] except KeyError: contentType = '' resp = response.read() # some commands return json if contentType == 'application/json': if resp: return json.loads(resp) return '' else: # some commands return plain text resp = makeUnicode(resp) logger.debug("QBitTorrent returned %s" % resp) if command == 'version/api': return resp # some just return Ok. or Fails. if resp and resp != 'Ok.': return False # some commands return nothing but response code (always 200) return True except URLError as err: logger.debug('Failed URL: %s' % url) logger.debug('QBitTorrent webUI raised the following error: %s' % err.reason) return False
def book_file(search_dir=None, booktype=None): # find a book/mag file in this directory, any book will do # return full pathname of book/mag, or empty string if none found if search_dir is None or booktype is None: return "" if search_dir and os.path.isdir(search_dir): try: for fname in os.listdir(makeBytestr(search_dir)): fname = makeUnicode(fname) if is_valid_booktype(fname, booktype=booktype): return os.path.join(search_dir, fname) except Exception as e: logger.warn('Listdir error [%s]: %s %s' % (search_dir, type(e).__name__, str(e))) return ""
def _command(self, command, args=None, content_type=None, files=None): logger.debug('QBittorrent WebAPI Command: %s' % command) url = self.base_url + '/' + command data = None headers = dict() if files or content_type == 'multipart/form-data': data, headers = encode_multipart(args, files, '-------------------------acebdf13572468') else: if args: data = makeBytestr(urlencode(args)) if content_type: headers['Content-Type'] = content_type request = Request(url, data, headers) if lazylibrarian.CONFIG['PROXY_HOST']: for item in getList(lazylibrarian.CONFIG['PROXY_TYPE']): request.set_proxy(lazylibrarian.CONFIG['PROXY_HOST'], item) request.add_header('User-Agent', getUserAgent()) try: response = self.opener.open(request) try: contentType = response.headers['content-type'] except KeyError: contentType = '' resp = response.read() # some commands return json if contentType == 'application/json': if resp: return json.loads(resp) return '' else: # some commands return plain text resp = makeUnicode(resp) logger.debug("QBitTorrent returned %s" % resp) if command == 'version/api': return resp # some just return Ok. or Fails. if resp and resp != 'Ok.': return False # some commands return nothing but response code (always 200) return True except URLError as err: logger.debug('Failed URL: %s' % url) logger.debug('QBitTorrent webUI raised the following error: %s' % err.reason) return False
def _listMissingWorkpages(self): # first the ones with no workpage q = 'SELECT BookID from books where length(WorkPage) < 4' res = self._dic_from_query(q) # now the ones with an error page cache = os.path.join(lazylibrarian.CACHEDIR, "WorkCache") if os.path.isdir(cache): for cached_file in os.listdir(makeBytestr(cache)): cached_file = makeUnicode(cached_file) target = os.path.join(cache, cached_file) if os.path.isfile(target): if os.path.getsize(target) < 500 and '.' in cached_file: bookid = cached_file.split('.')[0] res.append({"BookID": bookid}) self.data = res
def _send_tweet(self, message=None): username = self.consumer_key password = self.consumer_secret access_token_key = lazylibrarian.CONFIG['TWITTER_USERNAME'] access_token_secret = lazylibrarian.CONFIG['TWITTER_PASSWORD'] if not access_token_key or not access_token_secret: logger.error("No authorization found for twitter") return False logger.info("Sending tweet: " + message) api = twitter.Api(username, password, access_token_key, access_token_secret) message = formatter.makeBytestr(message) try: api.PostUpdate(message) except Exception as e: logger.error("Error Sending Tweet: %s" % e) return False return True
def opf_file(search_dir=None): if search_dir is None: return "" cnt = 0 res = '' meta = '' if os.path.isdir(search_dir): for fname in os.listdir(makeBytestr(search_dir)): fname = makeUnicode(fname) if fname.endswith('.opf'): if fname == 'metadata.opf': meta = os.path.join(search_dir, fname) else: res = os.path.join(search_dir, fname) cnt += 1 if cnt > 2 or cnt == 2 and not meta: logger.debug("Found %d conflicting opf in %s" % (cnt, search_dir)) res = '' elif res: # prefer bookname.opf over metadata.opf return res elif meta: return meta return res
def bookRename(bookid): myDB = database.DBConnection() cmd = 'select AuthorName,BookName,BookFile from books,authors where books.AuthorID = authors.AuthorID and bookid=?' exists = myDB.match(cmd, (bookid,)) if not exists: logger.debug("Invalid bookid in bookRename %s" % bookid) return '' f = exists['BookFile'] if not f: logger.debug("No filename for %s in BookRename %s" % bookid) return '' r = os.path.dirname(f) try: # noinspection PyTypeChecker calibreid = r.rsplit('(', 1)[1].split(')')[0] if not calibreid.isdigit(): calibreid = '' except IndexError: calibreid = '' if calibreid: msg = '[%s] looks like a calibre directory: not renaming book' % os.path.basename(r) logger.debug(msg) return f seriesinfo = seriesInfo(bookid) dest_path = lazylibrarian.CONFIG['EBOOK_DEST_FOLDER'].replace( '$Author', exists['AuthorName']).replace( '$Title', exists['BookName']).replace( '$Series', seriesinfo['Full']).replace( '$SerName', seriesinfo['Name']).replace( '$SerNum', seriesinfo['Num']).replace( '$$', ' ') dest_path = ' '.join(dest_path.split()).strip() dest_path = replace_all(dest_path, __dic__) dest_dir = lazylibrarian.DIRECTORY('eBook') dest_path = os.path.join(dest_dir, dest_path) if r != dest_path: try: dest_path = safe_move(r, dest_path) r = dest_path except Exception as why: if not os.path.isdir(dest_path): logger.error('Unable to create directory %s: %s' % (dest_path, why)) book_basename, prefextn = os.path.splitext(os.path.basename(f)) new_basename = lazylibrarian.CONFIG['EBOOK_DEST_FILE'] seriesinfo = seriesInfo(bookid) new_basename = new_basename.replace( '$Author', exists['AuthorName']).replace( '$Title', exists['BookName']).replace( '$Series', seriesinfo['Full']).replace( '$SerName', seriesinfo['Name']).replace( '$SerNum', seriesinfo['Num']).replace( '$$', ' ') new_basename = ' '.join(new_basename.split()).strip() # replace all '/' not surrounded by whitespace with '_' as '/' is a directory separator slash = new_basename.find('/') while slash > 0: if new_basename[slash - 1] != ' ': if new_basename[slash + 1] != ' ': new_basename = new_basename[:slash] + '_' + new_basename[slash + 1:] slash = new_basename.find('/', slash + 1) if ' / ' in new_basename: # used as a separator in goodreads omnibus logger.warn("bookRename [%s] looks like an omnibus? Not renaming %s" % (new_basename, book_basename)) new_basename = book_basename if book_basename != new_basename: # only rename bookname.type, bookname.jpg, bookname.opf, not cover.jpg or metadata.opf for fname in os.listdir(makeBytestr(r)): fname = makeUnicode(fname) extn = '' if is_valid_booktype(fname, booktype='ebook'): extn = os.path.splitext(fname)[1] elif fname.endswith('.opf') and not fname == 'metadata.opf': extn = '.opf' elif fname.endswith('.jpg') and not fname == 'cover.jpg': extn = '.jpg' if extn: ofname = os.path.join(r, fname) nfname = os.path.join(r, new_basename + extn) try: nfname = safe_move(ofname, nfname) logger.debug("bookRename %s to %s" % (ofname, nfname)) if ofname == exists['BookFile']: # if we renamed the preferred filetype, return new name f = nfname except Exception as e: logger.error('Unable to rename [%s] to [%s] %s %s' % (ofname, nfname, type(e).__name__, str(e))) return f
def cleanCache(): """ Remove unused files from the cache - delete if expired or unused. Check JSONCache WorkCache XMLCache SeriesCache Author Book Magazine Check covers and authorimages referenced in the database exist and change database entry if missing """ myDB = database.DBConnection() result = [] cache = os.path.join(lazylibrarian.CACHEDIR, "JSONCache") cleaned = 0 kept = 0 if os.path.isdir(cache): for cached_file in os.listdir(makeBytestr(cache)): cached_file = makeUnicode(cached_file) target = os.path.join(cache, cached_file) cache_modified_time = os.stat(target).st_mtime time_now = time.time() if cache_modified_time < time_now - ( lazylibrarian.CONFIG['CACHE_AGE'] * 24 * 60 * 60): # expire after this many seconds # Cache is old, delete entry os.remove(target) cleaned += 1 else: kept += 1 msg = "Cleaned %i file%s from JSONCache, kept %i" % (cleaned, plural(cleaned), kept) result.append(msg) logger.debug(msg) cache = os.path.join(lazylibrarian.CACHEDIR, "XMLCache") cleaned = 0 kept = 0 if os.path.isdir(cache): for cached_file in os.listdir(makeBytestr(cache)): cached_file = makeUnicode(cached_file) target = os.path.join(cache, cached_file) cache_modified_time = os.stat(target).st_mtime time_now = time.time() if cache_modified_time < time_now - ( lazylibrarian.CONFIG['CACHE_AGE'] * 24 * 60 * 60): # expire after this many seconds # Cache is old, delete entry os.remove(target) cleaned += 1 else: kept += 1 msg = "Cleaned %i file%s from XMLCache, kept %i" % (cleaned, plural(cleaned), kept) result.append(msg) logger.debug(msg) cache = os.path.join(lazylibrarian.CACHEDIR, "WorkCache") cleaned = 0 kept = 0 if os.path.isdir(cache): for cached_file in os.listdir(makeBytestr(cache)): cached_file = makeUnicode(cached_file) target = os.path.join(cache, cached_file) try: bookid = cached_file.split('.')[0] except IndexError: logger.error('Clean Cache: Error splitting %s' % cached_file) continue item = myDB.match('select BookID from books where BookID=?', (bookid,)) if not item: # WorkPage no longer referenced in database, delete cached_file os.remove(target) cleaned += 1 else: kept += 1 msg = "Cleaned %i file%s from WorkCache, kept %i" % (cleaned, plural(cleaned), kept) result.append(msg) logger.debug(msg) cache = os.path.join(lazylibrarian.CACHEDIR, "SeriesCache") cleaned = 0 kept = 0 if os.path.isdir(cache): for cached_file in os.listdir(makeBytestr(cache)): cached_file = makeUnicode(cached_file) target = os.path.join(cache, cached_file) try: seriesid = cached_file.split('.')[0] except IndexError: logger.error('Clean Cache: Error splitting %s' % cached_file) continue item = myDB.match('select SeriesID from series where SeriesID=?', (seriesid,)) if not item: # SeriesPage no longer referenced in database, delete cached_file os.remove(target) cleaned += 1 else: kept += 1 msg = "Cleaned %i file%s from SeriesCache, kept %i" % (cleaned, plural(cleaned), kept) result.append(msg) logger.debug(msg) cache = os.path.join(lazylibrarian.CACHEDIR, "magazine") cleaned = 0 kept = 0 if os.path.isdir(cache): # we can clear the magazine cache, it gets rebuilt as required # this does not delete our magazine cover files, only the small cached copy for cached_file in os.listdir(makeBytestr(cache)): cached_file = makeUnicode(cached_file) target = os.path.join(cache, cached_file) if target.endswith('.jpg'): os.remove(target) cleaned += 1 else: kept += 1 msg = "Cleaned %i file%s from magazine cache, kept %i" % (cleaned, plural(cleaned), kept) result.append(msg) logger.debug(msg) cache = lazylibrarian.CACHEDIR cleaned = 0 kept = 0 cachedir = os.path.join(cache, 'author') if os.path.isdir(cachedir): for cached_file in os.listdir(makeBytestr(cachedir)): cached_file = makeUnicode(cached_file) target = os.path.join(cachedir, cached_file) if os.path.isfile(target): try: imgid = cached_file.split('.')[0].rsplit(os.sep)[-1] except IndexError: logger.error('Clean Cache: Error splitting %s' % cached_file) continue item = myDB.match('select AuthorID from authors where AuthorID=?', (imgid,)) if not item: # Author Image no longer referenced in database, delete cached_file os.remove(target) cleaned += 1 else: kept += 1 cachedir = os.path.join(cache, 'book') if os.path.isdir(cachedir): for cached_file in os.listdir(makeBytestr(cachedir)): cached_file = makeUnicode(cached_file) target = os.path.join(cachedir, cached_file) if os.path.isfile(target): try: imgid = cached_file.split('.')[0].rsplit(os.sep)[-1] except IndexError: logger.error('Clean Cache: Error splitting %s' % cached_file) continue item = myDB.match('select BookID from books where BookID=?', (imgid,)) if not item: # Book Image no longer referenced in database, delete cached_file os.remove(target) cleaned += 1 else: kept += 1 # at this point there should be no more .jpg files in the root of the cachedir # any that are still there are for books/authors deleted from database for cached_file in os.listdir(makeBytestr(cache)): cached_file = makeUnicode(cached_file) if cached_file.endswith('.jpg'): os.remove(os.path.join(cache, cached_file)) cleaned += 1 msg = "Cleaned %i file%s from ImageCache, kept %i" % (cleaned, plural(cleaned), kept) result.append(msg) logger.debug(msg) # verify the cover images referenced in the database are present images = myDB.action('select BookImg,BookName,BookID from books') cachedir = os.path.join(lazylibrarian.CACHEDIR, 'book') cleaned = 0 kept = 0 for item in images: keep = True imgfile = '' if item['BookImg'] is None or item['BookImg'] == '': keep = False if keep and not item['BookImg'].startswith('http') and not item['BookImg'] == "images/nocover.png": # html uses '/' as separator, but os might not imgname = item['BookImg'].rsplit('/')[-1] imgfile = os.path.join(cachedir, imgname) if not os.path.isfile(imgfile): keep = False if keep: kept += 1 else: cleaned += 1 logger.debug('Cover missing for %s %s' % (item['BookName'], imgfile)) myDB.action('update books set BookImg="images/nocover.png" where Bookid=?', (item['BookID'],)) msg = "Cleaned %i missing cover file%s, kept %i" % (cleaned, plural(cleaned), kept) result.append(msg) logger.debug(msg) # verify the author images referenced in the database are present images = myDB.action('select AuthorImg,AuthorName,AuthorID from authors') cachedir = os.path.join(lazylibrarian.CACHEDIR, 'author') cleaned = 0 kept = 0 for item in images: keep = True imgfile = '' if item['AuthorImg'] is None or item['AuthorImg'] == '': keep = False if keep and not item['AuthorImg'].startswith('http') and not item['AuthorImg'] == "images/nophoto.png": # html uses '/' as separator, but os might not imgname = item['AuthorImg'].rsplit('/')[-1] imgfile = os.path.join(cachedir, imgname) if not os.path.isfile(imgfile): keep = False if keep: kept += 1 else: cleaned += 1 logger.debug('Image missing for %s %s' % (item['AuthorName'], imgfile)) myDB.action('update authors set AuthorImg="images/nophoto.png" where AuthorID=?', (item['AuthorID'],)) msg = "Cleaned %i missing author image%s, kept %i" % (cleaned, plural(cleaned), kept) result.append(msg) logger.debug(msg) return result
def bookRename(bookid): myDB = database.DBConnection() cmd = 'select AuthorName,BookName,BookFile from books,authors where books.AuthorID = authors.AuthorID and bookid=?' exists = myDB.match(cmd, (bookid,)) if not exists: logger.debug("Invalid bookid in bookRename %s" % bookid) return '' f = exists['BookFile'] if not f: logger.debug("No filename for %s in BookRename %s" % bookid) return '' r = os.path.dirname(f) if not lazylibrarian.CONFIG['CALIBRE_RENAME']: try: # noinspection PyTypeChecker calibreid = r.rsplit('(', 1)[1].split(')')[0] if not calibreid.isdigit(): calibreid = '' except IndexError: calibreid = '' if calibreid: msg = '[%s] looks like a calibre directory: not renaming book' % os.path.basename(r) logger.debug(msg) return f reject = multibook(r) if reject: logger.debug("Not renaming %s, found multiple %s" % (f, reject)) return f seriesinfo = nameVars(bookid) dest_path = seriesinfo['FolderName'] dest_dir = lazylibrarian.DIRECTORY('eBook') dest_path = os.path.join(dest_dir, dest_path) dest_path = stripspaces(dest_path) oldpath = r if oldpath != dest_path: try: dest_path = safe_move(oldpath, dest_path) except Exception as why: if not os.path.isdir(dest_path): logger.error('Unable to create directory %s: %s' % (dest_path, why)) book_basename, prefextn = os.path.splitext(os.path.basename(f)) new_basename = seriesinfo['BookFile'] if ' / ' in new_basename: # used as a separator in goodreads omnibus logger.warn("bookRename [%s] looks like an omnibus? Not renaming %s" % (new_basename, book_basename)) new_basename = book_basename if book_basename != new_basename: # only rename bookname.type, bookname.jpg, bookname.opf, not cover.jpg or metadata.opf for fname in os.listdir(makeBytestr(dest_path)): fname = makeUnicode(fname) extn = '' if is_valid_booktype(fname, booktype='ebook'): extn = os.path.splitext(fname)[1] elif fname.endswith('.opf') and not fname == 'metadata.opf': extn = '.opf' elif fname.endswith('.jpg') and not fname == 'cover.jpg': extn = '.jpg' if extn: ofname = os.path.join(dest_path, fname) nfname = os.path.join(dest_path, new_basename + extn) if ofname != nfname: try: nfname = safe_move(ofname, nfname) logger.debug("bookRename %s to %s" % (ofname, nfname)) oldname = os.path.join(oldpath, fname) if oldname == exists['BookFile']: # if we renamed/moved the preferred file, return new name f = nfname except Exception as e: logger.error('Unable to rename [%s] to [%s] %s %s' % (ofname, nfname, type(e).__name__, str(e))) return f
def audioProcess(bookid, rename=False, playlist=False): """ :param bookid: book to process :param rename: rename to match audiobook filename pattern :param playlist: generate a playlist for popup :return: filename of part 01 of the audiobook """ for item in ['$Part', '$Title']: if rename and item not in lazylibrarian.CONFIG['AUDIOBOOK_DEST_FILE']: logger.error("Unable to audioProcess, check AUDIOBOOK_DEST_FILE") return '' myDB = database.DBConnection() cmd = 'select AuthorName,BookName,AudioFile from books,authors where books.AuthorID = authors.AuthorID and bookid=?' exists = myDB.match(cmd, (bookid,)) if exists: book_filename = exists['AudioFile'] if book_filename: r = os.path.dirname(book_filename) else: logger.debug("No filename for %s in audioProcess" % bookid) return '' else: logger.debug("Invalid bookid in audioProcess %s" % bookid) return '' if not TinyTag: logger.warn("TinyTag library not available") return '' cnt = 0 parts = [] total = 0 author = '' book = '' audio_file = '' abridged = '' for f in os.listdir(makeBytestr(r)): f = makeUnicode(f) if is_valid_booktype(f, booktype='audiobook'): cnt += 1 audio_file = f try: audio_path = os.path.join(r, f) performer = '' composer = '' albumartist = '' book = '' title = '' track = 0 total = 0 if TinyTag.is_supported(audio_path): id3r = TinyTag.get(audio_path) performer = id3r.artist composer = id3r.composer albumartist = id3r.albumartist book = id3r.album title = id3r.title track = id3r.track total = id3r.track_total track = check_int(track, 0) total = check_int(total, 0) if performer: performer = performer.strip() if composer: composer = composer.strip() if book: book = book.strip() if albumartist: albumartist = albumartist.strip() if composer: # if present, should be author author = composer elif performer: # author, or narrator if composer == author author = performer elif albumartist: author = albumartist if author and book: parts.append([track, book, author, f]) if not abridged: for tag in [book, title, albumartist, performer, composer]: if tag and 'unabridged' in tag.lower(): abridged = 'Unabridged' break if not abridged: for tag in [book, title, albumartist, performer, composer]: if tag and 'abridged' in tag.lower(): abridged = 'Abridged' break except Exception as e: logger.error("tinytag %s %s" % (type(e).__name__, str(e))) pass finally: if not abridged: if audio_file and 'unabridged' in audio_file.lower(): abridged = 'Unabridged' break if not abridged: if audio_file and 'abridged' in audio_file.lower(): abridged = 'Abridged' break logger.debug("%s found %s audiofile%s" % (exists['BookName'], cnt, plural(cnt))) if cnt == 1 and not parts: # single file audiobook with no tags parts = [[1, exists['BookName'], exists['AuthorName'], audio_file]] if cnt != len(parts): logger.warn("%s: Incorrect number of parts (found %i from %i)" % (exists['BookName'], len(parts), cnt)) return book_filename if total and total != cnt: logger.warn("%s: Reported %i parts, got %i" % (exists['BookName'], total, cnt)) return book_filename # check all parts have the same author and title if len(parts) > 1: for part in parts: if part[1] != book: logger.warn("%s: Inconsistent title: [%s][%s]" % (exists['BookName'], part[1], book)) return book_filename if part[2] != author: logger.warn("%s: Inconsistent author: [%s][%s]" % (exists['BookName'], part[2], author)) return book_filename # do we have any track info (value is 0 if not) if parts[0][0] == 0: tokmatch = '' # try to extract part information from filename. Search for token style of part 1 in this order... for token in [' 001.', ' 01.', ' 1.', ' 001 ', ' 01 ', ' 1 ', '01']: if tokmatch: break for part in parts: if token in part[3]: tokmatch = token break if tokmatch: # we know the numbering style, get numbers for the other parts cnt = 0 while cnt < len(parts): cnt += 1 if tokmatch == ' 001.': pattern = ' %s.' % str(cnt).zfill(3) elif tokmatch == ' 01.': pattern = ' %s.' % str(cnt).zfill(2) elif tokmatch == ' 1.': pattern = ' %s.' % str(cnt) elif tokmatch == ' 001 ': pattern = ' %s ' % str(cnt).zfill(3) elif tokmatch == ' 01 ': pattern = ' %s ' % str(cnt).zfill(2) elif tokmatch == ' 1 ': pattern = ' %s ' % str(cnt) else: pattern = '%s' % str(cnt).zfill(2) # standardise numbering of the parts for part in parts: if pattern in part[3]: part[0] = cnt break parts.sort(key=lambda x: x[0]) # check all parts are present cnt = 0 while cnt < len(parts): if parts[cnt][0] != cnt + 1: logger.warn("%s: No part %i found" % (exists['BookName'], cnt + 1)) return book_filename cnt += 1 if abridged: abridged = ' (%s)' % abridged # if we get here, looks like we have all the parts needed to rename properly seriesinfo = nameVars(bookid, abridged) dest_path = seriesinfo['FolderName'] dest_dir = lazylibrarian.DIRECTORY('Audio') dest_path = os.path.join(dest_dir, dest_path) if rename and r != dest_path: try: dest_path = safe_move(r, dest_path) r = dest_path except Exception as why: if not os.path.isdir(dest_path): logger.error('Unable to create directory %s: %s' % (dest_path, why)) if playlist: try: playlist = open(os.path.join(r, 'playlist.ll'), 'w') except Exception as why: logger.error('Unable to create playlist in %s: %s' % (r, why)) playlist = None for part in parts: pattern = seriesinfo['AudioFile'] pattern = pattern.replace( '$Part', str(part[0]).zfill(len(str(len(parts))))).replace( '$Total', str(len(parts))) pattern = ' '.join(pattern.split()).strip() pattern = pattern + os.path.splitext(part[3])[1] if playlist: if rename: playlist.write(pattern + '\n') else: playlist.write(part[3] + '\n') if rename: n = os.path.join(r, pattern) o = os.path.join(r, part[3]) if o != n: try: n = safe_move(o, n) if part[0] == 1: book_filename = n # return part 1 of set logger.debug('%s: audioProcess [%s] to [%s]' % (exists['BookName'], o, n)) except Exception as e: logger.error('Unable to rename [%s] to [%s] %s %s' % (o, n, type(e).__name__, str(e))) if playlist: playlist.close() return book_filename
def createMagCover(issuefile=None, refresh=False): if not lazylibrarian.CONFIG['IMP_MAGCOVER']: return if issuefile is None or not os.path.isfile(issuefile): logger.debug('No issuefile %s' % issuefile) return base, extn = os.path.splitext(issuefile) if not extn: logger.debug('Unable to create cover for %s, no extension?' % issuefile) return coverfile = base + '.jpg' if os.path.isfile(coverfile): if refresh: os.remove(coverfile) else: logger.debug('Cover for %s exists' % issuefile) return # quit if cover already exists and we didn't want to refresh logger.debug('Creating cover for %s' % issuefile) data = '' # result from unzip or unrar extn = extn.lower() if extn in ['.cbz', '.epub']: try: data = zipfile.ZipFile(issuefile) except Exception as why: logger.error("Failed to read zip file %s, %s %s" % (issuefile, type(why).__name__, str(why))) data = '' elif extn in ['.cbr']: try: # unrar will complain if the library isn't installed, needs to be compiled separately # see https://pypi.python.org/pypi/unrar/ for instructions # Download source from http://www.rarlab.com/rar_add.htm # note we need LIBRARY SOURCE not a binary package # make lib; sudo make install-lib; sudo ldconfig # lib.unrar should then be able to find libunrar.so from lib.unrar import rarfile data = rarfile.RarFile(issuefile) except Exception as why: logger.error("Failed to read rar file %s, %s %s" % (issuefile, type(why).__name__, str(why))) data = '' if data: img = None try: for member in data.namelist(): memlow = member.lower() if '-00.' in memlow or '000.' in memlow or 'cover.' in memlow: if memlow.endswith('.jpg') or memlow.endswith('.jpeg'): img = data.read(member) break if img: with open(coverfile, 'wb') as f: if PY2: f.write(img) else: f.write(img.encode()) return else: logger.debug("Failed to find image in %s" % issuefile) except Exception as why: logger.error("Failed to extract image from %s, %s %s" % (issuefile, type(why).__name__, str(why))) elif extn == '.pdf': generator = "" if len(lazylibrarian.CONFIG['IMP_CONVERT']): # allow external convert to override libraries generator = "external program: %s" % lazylibrarian.CONFIG['IMP_CONVERT'] if "gsconvert.py" in lazylibrarian.CONFIG['IMP_CONVERT']: msg = "Use of gsconvert.py is deprecated, equivalent functionality is now built in. " msg += "Support for gsconvert.py may be removed in a future release. See wiki for details." logger.warn(msg) converter = lazylibrarian.CONFIG['IMP_CONVERT'] postfix = '' # if not os.path.isfile(converter): # full path given, or just program_name? # converter = os.path.join(os.getcwd(), lazylibrarian.CONFIG['IMP_CONVERT']) if 'convert' in converter and 'gs' not in converter: # tell imagemagick to only convert first page postfix = '[0]' try: params = [converter, '%s%s' % (issuefile, postfix), '%s' % coverfile] res = subprocess.check_output(params, stderr=subprocess.STDOUT) res = makeUnicode(res).strip() if res: logger.debug('%s reports: %s' % (lazylibrarian.CONFIG['IMP_CONVERT'], res)) except Exception as e: # logger.debug(params) logger.warn('External "convert" failed %s %s' % (type(e).__name__, str(e))) elif platform.system() == "Windows": GS = os.path.join(os.getcwd(), "gswin64c.exe") generator = "local gswin64c" if not os.path.isfile(GS): GS = os.path.join(os.getcwd(), "gswin32c.exe") generator = "local gswin32c" if not os.path.isfile(GS): params = ["where", "gswin64c"] try: GS = subprocess.check_output(params, stderr=subprocess.STDOUT) GS = makeUnicode(GS).strip() generator = "gswin64c" except Exception as e: logger.debug("where gswin64c failed: %s %s" % (type(e).__name__, str(e))) if not os.path.isfile(GS): params = ["where", "gswin32c"] try: GS = subprocess.check_output(params, stderr=subprocess.STDOUT) GS = makeUnicode(GS).strip() generator = "gswin32c" except Exception as e: logger.debug("where gswin32c failed: %s %s" % (type(e).__name__, str(e))) if not os.path.isfile(GS): logger.debug("No gswin found") generator = "(no windows ghostscript found)" else: # noinspection PyBroadException try: params = [GS, "--version"] res = subprocess.check_output(params, stderr=subprocess.STDOUT) res = makeUnicode(res).strip() logger.debug("Found %s [%s] version %s" % (generator, GS, res)) generator = "%s version %s" % (generator, res) issuefile = issuefile.split('[')[0] params = [GS, "-sDEVICE=jpeg", "-dNOPAUSE", "-dBATCH", "-dSAFER", "-dFirstPage=1", "-dLastPage=1", "-dUseCropBox", "-sOutputFile=%s" % coverfile, issuefile] res = subprocess.check_output(params, stderr=subprocess.STDOUT) res = makeUnicode(res).strip() if not os.path.isfile(coverfile): logger.debug("Failed to create jpg: %s" % res) except Exception: # as why: logger.warn("Failed to create jpg for %s" % issuefile) logger.debug('Exception in gswin create_cover: %s' % traceback.format_exc()) else: # not windows try: # noinspection PyUnresolvedReferences from wand.image import Image interface = "wand" except ImportError: try: # No PythonMagick in python3 # noinspection PyUnresolvedReferences import PythonMagick interface = "pythonmagick" except ImportError: interface = "" try: if interface == 'wand': generator = "wand interface" with Image(filename=issuefile + '[0]') as img: img.save(filename=coverfile) elif interface == 'pythonmagick': generator = "pythonmagick interface" img = PythonMagick.Image() # PythonMagick requires filenames to be bytestr, not unicode if type(issuefile) is text_type: issuefile = makeBytestr(issuefile) if type(coverfile) is text_type: coverfile = makeBytestr(coverfile) img.read(issuefile + '[0]') img.write(coverfile) else: GS = os.path.join(os.getcwd(), "gs") generator = "local gs" if not os.path.isfile(GS): GS = "" params = ["which", "gs"] try: GS = subprocess.check_output(params, stderr=subprocess.STDOUT) GS = makeUnicode(GS).strip() generator = GS except Exception as e: logger.debug("which gs failed: %s %s" % (type(e).__name__, str(e))) if not os.path.isfile(GS): logger.debug("Cannot find gs") generator = "(no gs found)" else: params = [GS, "--version"] res = subprocess.check_output(params, stderr=subprocess.STDOUT) res = makeUnicode(res).strip() logger.debug("Found gs [%s] version %s" % (GS, res)) generator = "%s version %s" % (generator, res) issuefile = issuefile.split('[')[0] params = [GS, "-sDEVICE=jpeg", "-dNOPAUSE", "-dBATCH", "-dSAFER", "-dFirstPage=1", "-dLastPage=1", "-dUseCropBox", "-sOutputFile=%s" % coverfile, issuefile] res = subprocess.check_output(params, stderr=subprocess.STDOUT) res = makeUnicode(res).strip() if not os.path.isfile(coverfile): logger.debug("Failed to create jpg: %s" % res) except Exception as e: logger.warn("Unable to create cover for %s using %s %s" % (issuefile, type(e).__name__, generator)) logger.debug('Exception in create_cover: %s' % traceback.format_exc()) if os.path.isfile(coverfile): setperm(coverfile) logger.debug("Created cover for %s using %s" % (issuefile, generator)) return # if not recognised extension or cover creation failed try: coverfile = safe_copy(os.path.join(lazylibrarian.PROG_DIR, 'data/images/nocover.jpg'), coverfile) setperm(coverfile) except Exception as why: logger.error("Failed to copy nocover file, %s %s" % (type(why).__name__, str(why))) return
def create_id(issuename=None): hashID = sha1(makeBytestr(issuename)).hexdigest() # logger.debug('Issue %s Hash: %s' % (issuename, hashID)) return hashID
def db_v14(myDB, upgradelog): upgradelog.write("%s v14: %s\n" % (time.ctime(), "Moving image caches")) src = lazylibrarian.CACHEDIR try: os.mkdir(os.path.join(src, 'author')) except OSError as e: if e.errno not in [17, 183]: # already exists is ok msg = 'mkdir author cache reports: %s' % str(e) logger.debug(msg) upgradelog.write("%s v14: %s\n" % (time.ctime(), msg)) query = 'SELECT AuthorName, AuthorID, AuthorImg FROM authors ' query += 'WHERE AuthorImg LIKE "cache/%" ' query += 'AND AuthorImg NOT LIKE "cache/author/%"' images = myDB.select(query) if images: tot = len(images) msg = 'Moving %s author images to new location' % tot logger.debug(msg) upgradelog.write("%s v14: %s\n" % (time.ctime(), msg)) cnt = 0 for image in images: cnt += 1 lazylibrarian.UPDATE_MSG = "Moving author images to new location: %s of %s" % (cnt, tot) try: img = image['AuthorImg'] img = img.rsplit('/', 1)[1] srcfile = os.path.join(src, img) if os.path.isfile(srcfile): try: shutil.move(srcfile, os.path.join(src, "author", img)) myDB.action( 'UPDATE authors SET AuthorImg="cache/author/?" WHERE AuthorID=?', (img, image['AuthorID'])) except Exception as e: logger.error("dbupgrade: %s %s" % (type(e).__name__, str(e))) except Exception as e: msg = 'Failed to update author image for %s: %s %s' % (image['AuthorName'], type(e).__name__, str(e)) logger.warn(msg) upgradelog.write("%s v14: %s\n" % (time.ctime(), msg)) upgradelog.write("%s v14: %s\n" % (time.ctime(), lazylibrarian.UPDATE_MSG)) logger.debug("Author Image cache updated") try: os.mkdir(os.path.join(src, 'book')) except OSError as e: if e.errno not in [17, 183]: # already exists is ok msg = 'mkdir book cache reports: %s' % str(e) logger.debug(msg) upgradelog.write("%s v14: %s\n" % (time.ctime(), msg)) query = 'SELECT BookName, BookID, BookImg FROM books ' query += 'WHERE BookImg LIKE "cache/%" ' query += 'AND BookImg NOT LIKE "cache/book/%"' images = myDB.select(query) if images: tot = len(images) msg = 'Moving %s book images to new location' % tot upgradelog.write("%s v14: %s\n" % (time.ctime(), msg)) logger.debug(msg) cnt = 0 for image in images: cnt += 1 lazylibrarian.UPDATE_MSG = "Moving book images to new location: %s of %s" % (cnt, tot) try: img = image['BookImg'] img = img.rsplit('/', 1)[1] srcfile = os.path.join(src, img) if os.path.isfile(srcfile): try: shutil.move(srcfile, os.path.join(src, "book", img)) myDB.action('UPDATE books SET BookImg="cache/book/?" WHERE BookID=?', (img, image['BookID'])) except Exception as e: logger.error("dbupgrade: %s %s" % (type(e).__name__, str(e))) except Exception as e: logger.warn('Failed to update book image for %s: %s %s' % (image['BookName'], type(e).__name__, str(e))) upgradelog.write("%s v14: %s\n" % (time.ctime(), lazylibrarian.UPDATE_MSG)) logger.debug("Book Image cache updated") # at this point there should be no more .jpg files in the root of the cachedir # any that are still there are for books/authors deleted from database # or magazine latest issue cover files that get copied as required for image in os.listdir(makeBytestr(src)): image = makeUnicode(image) if image.endswith('.jpg'): os.remove(os.path.join(src, image)) upgradelog.write("%s v14: complete\n" % time.ctime())
def magazineScan(title=None): lazylibrarian.MAG_UPDATE = 1 # noinspection PyBroadException try: myDB = database.DBConnection() onetitle = title if onetitle: mag_path = lazylibrarian.CONFIG['MAG_DEST_FOLDER'].replace( '$Title', onetitle) else: mag_path = os.path.dirname(lazylibrarian.CONFIG['MAG_DEST_FOLDER']) if lazylibrarian.CONFIG['MAG_RELATIVE']: mag_path = os.path.join(lazylibrarian.DIRECTORY('eBook'), mag_path) if PY2: mag_path = mag_path.encode(lazylibrarian.SYS_ENCODING) if lazylibrarian.CONFIG['FULL_SCAN'] and not onetitle: mags = myDB.select('select * from Issues') # check all the issues are still there, delete entry if not for mag in mags: title = mag['Title'] issuedate = mag['IssueDate'] issuefile = mag['IssueFile'] if issuefile and not os.path.isfile(issuefile): myDB.action('DELETE from Issues where issuefile=?', (issuefile, )) logger.info('Issue %s - %s deleted as not found on disk' % (title, issuedate)) controlValueDict = {"Title": title} newValueDict = { "LastAcquired": None, # clear magazine dates "IssueDate": None, # we will fill them in again later "LatestCover": None, "IssueStatus": "Skipped" # assume there are no issues now } myDB.upsert("magazines", newValueDict, controlValueDict) logger.debug('Magazine %s details reset' % title) # now check the magazine titles and delete any with no issues if lazylibrarian.CONFIG['MAG_DELFOLDER']: mags = myDB.select( 'SELECT Title,count(Title) as counter from issues group by Title' ) for mag in mags: title = mag['Title'] issues = mag['counter'] if not issues: logger.debug('Magazine %s deleted as no issues found' % title) myDB.action('DELETE from magazines WHERE Title=?', (title, )) logger.info(' Checking [%s] for magazines' % mag_path) matchString = '' for char in lazylibrarian.CONFIG['MAG_DEST_FILE']: matchString = matchString + '\\' + char # massage the MAG_DEST_FILE config parameter into something we can use # with regular expression matching booktypes = '' count = -1 booktype_list = getList(lazylibrarian.CONFIG['MAG_TYPE']) for book_type in booktype_list: count += 1 if count == 0: booktypes = book_type else: booktypes = booktypes + '|' + book_type match = matchString.replace( "\\$\\I\\s\\s\\u\\e\\D\\a\\t\\e", "(?P<issuedate>.*?)").replace( "\\$\\T\\i\\t\\l\\e", "(?P<title>.*?)") + '\.[' + booktypes + ']' title_pattern = re.compile(match, re.VERBOSE) match = matchString.replace( "\\$\\I\\s\\s\\u\\e\\D\\a\\t\\e", "(?P<issuedate>.*?)").replace( "\\$\\T\\i\\t\\l\\e", "") + '\.[' + booktypes + ']' date_pattern = re.compile(match, re.VERBOSE) # try to ensure startdir is str as os.walk can fail if it tries to convert a subdir or file # to utf-8 and fails (eg scandinavian characters in ascii 8bit) for rootdir, dirnames, filenames in os.walk(makeBytestr(mag_path)): rootdir = makeUnicode(rootdir) filenames = [makeUnicode(item) for item in filenames] for fname in filenames: # maybe not all magazines will be pdf? if is_valid_booktype(fname, booktype='mag'): issuedate = '' # noinspection PyBroadException try: match = title_pattern.match(fname) if match: title = match.group("title") issuedate = match.group("issuedate") if lazylibrarian.LOGLEVEL & lazylibrarian.log_magdates: logger.debug("Title pattern [%s][%s]" % (title, issuedate)) match = True else: logger.debug( "Title pattern match failed for [%s]" % fname) except Exception: match = False if not match: # noinspection PyBroadException try: match = date_pattern.match(fname) if match: issuedate = match.group("issuedate") title = os.path.basename(rootdir) if lazylibrarian.LOGLEVEL & lazylibrarian.log_magdates: logger.debug("Date pattern [%s][%s]" % (title, issuedate)) match = True else: logger.debug( "Date pattern match failed for [%s]" % fname) except Exception: match = False if not match: title = os.path.basename(rootdir) issuedate = '' dic = { '.': ' ', '-': ' ', '/': ' ', '+': ' ', '_': ' ', '(': '', ')': '', '[': ' ', ']': ' ', '#': '# ' } if issuedate: exploded = replace_all(issuedate, dic).split() regex_pass, issuedate, year = lazylibrarian.searchmag.get_issue_date( exploded) if lazylibrarian.LOGLEVEL & lazylibrarian.log_magdates: logger.debug("Date regex [%s][%s][%s]" % (regex_pass, issuedate, year)) if not regex_pass: issuedate = '' if not issuedate: exploded = replace_all(fname, dic).split() regex_pass, issuedate, year = lazylibrarian.searchmag.get_issue_date( exploded) if lazylibrarian.LOGLEVEL & lazylibrarian.log_magdates: logger.debug("File regex [%s][%s][%s]" % (regex_pass, issuedate, year)) if not regex_pass: issuedate = '' if not issuedate: logger.warn("Invalid name format for [%s]" % fname) continue issuefile = os.path.join(rootdir, fname) # full path to issue.pdf mtime = os.path.getmtime(issuefile) iss_acquired = datetime.date.isoformat( datetime.date.fromtimestamp(mtime)) if lazylibrarian.CONFIG['MAG_RENAME']: filedate = issuedate if issuedate and issuedate.isdigit(): if len(issuedate) == 8: if check_year(issuedate[:4]): filedate = 'Issue %d %s' % (int( issuedate[4:]), issuedate[:4]) else: filedate = 'Vol %d Iss %d' % (int( issuedate[:4]), int(issuedate[4:])) elif len(issuedate) == 12: filedate = 'Vol %d Iss %d %s' % (int( issuedate[4:8]), int( issuedate[8:]), issuedate[:4]) else: filedate = str(issuedate).zfill(4) extn = os.path.splitext(fname)[1] newfname = lazylibrarian.CONFIG[ 'MAG_DEST_FILE'].replace('$Title', title).replace( '$IssueDate', filedate) newfname = newfname + extn if newfname and newfname != fname: logger.debug("Rename %s -> %s" % (fname, newfname)) newissuefile = os.path.join(rootdir, newfname) newissuefile = safe_move(issuefile, newissuefile) if os.path.exists(issuefile.replace(extn, '.jpg')): safe_move(issuefile.replace(extn, '.jpg'), newissuefile.replace(extn, '.jpg')) if os.path.exists(issuefile.replace(extn, '.opf')): safe_move(issuefile.replace(extn, '.opf'), newissuefile.replace(extn, '.opf')) issuefile = newissuefile logger.debug("Found %s Issue %s" % (title, issuedate)) controlValueDict = {"Title": title} # is this magazine already in the database? mag_entry = myDB.match( 'SELECT LastAcquired,IssueDate,MagazineAdded,CoverPage from magazines WHERE Title=?', (title, )) if not mag_entry: # need to add a new magazine to the database newValueDict = { "Reject": None, "Status": "Active", "MagazineAdded": None, "LastAcquired": None, "LatestCover": None, "IssueDate": None, "IssueStatus": "Skipped", "Regex": None, "CoverPage": 1 } logger.debug("Adding magazine %s" % title) myDB.upsert("magazines", newValueDict, controlValueDict) magissuedate = None magazineadded = None maglastacquired = None magcoverpage = 1 else: maglastacquired = mag_entry['LastAcquired'] magissuedate = mag_entry['IssueDate'] magazineadded = mag_entry['MagazineAdded'] magissuedate = str(magissuedate).zfill(4) magcoverpage = mag_entry['CoverPage'] issuedate = str(issuedate).zfill( 4) # for sorting issue numbers # is this issue already in the database? issue_id = create_id("%s %s" % (title, issuedate)) iss_entry = myDB.match( 'SELECT Title,IssueFile from issues WHERE Title=? and IssueDate=?', (title, issuedate)) new_entry = False if not iss_entry or iss_entry['IssueFile'] != issuefile: new_entry = True # new entry or name changed if not iss_entry: logger.debug("Adding issue %s %s" % (title, issuedate)) else: logger.debug("Updating issue %s %s" % (title, issuedate)) controlValueDict = { "Title": title, "IssueDate": issuedate } newValueDict = { "IssueAcquired": iss_acquired, "IssueID": issue_id, "IssueFile": issuefile } myDB.upsert("Issues", newValueDict, controlValueDict) ignorefile = os.path.join(os.path.dirname(issuefile), '.ll_ignore') with open(ignorefile, 'a'): os.utime(ignorefile, None) createMagCover(issuefile, pagenum=magcoverpage, refresh=new_entry) lazylibrarian.postprocess.processMAGOPF( issuefile, title, issuedate, issue_id, overwrite=new_entry) # see if this issues date values are useful controlValueDict = {"Title": title} if not mag_entry: # new magazine, this is the only issue newValueDict = { "MagazineAdded": iss_acquired, "LastAcquired": iss_acquired, "LatestCover": os.path.splitext(issuefile)[0] + '.jpg', "IssueDate": issuedate, "IssueStatus": "Open" } myDB.upsert("magazines", newValueDict, controlValueDict) else: # Set magazine_issuedate to issuedate of most recent issue we have # Set latestcover to most recent issue cover # Set magazine_added to acquired date of earliest issue we have # Set magazine_lastacquired to acquired date of most recent issue we have # acquired dates are read from magazine file timestamps newValueDict = {"IssueStatus": "Open"} if not magazineadded or iss_acquired < magazineadded: newValueDict["MagazineAdded"] = iss_acquired if not maglastacquired or iss_acquired > maglastacquired: newValueDict["LastAcquired"] = iss_acquired if not magissuedate or issuedate >= magissuedate: newValueDict["IssueDate"] = issuedate newValueDict["LatestCover"] = os.path.splitext( issuefile)[0] + '.jpg' myDB.upsert("magazines", newValueDict, controlValueDict) if lazylibrarian.CONFIG['FULL_SCAN'] and not onetitle: magcount = myDB.match("select count(*) from magazines") isscount = myDB.match("select count(*) from issues") logger.info( "Magazine scan complete, found %s magazine%s, %s issue%s" % (magcount['count(*)'], plural(magcount['count(*)']), isscount['count(*)'], plural(isscount['count(*)']))) else: logger.info("Magazine scan complete") lazylibrarian.MAG_UPDATE = 0 except Exception: lazylibrarian.MAG_UPDATE = 0 logger.error('Unhandled exception in magazineScan: %s' % traceback.format_exc())
def magazineScan(title=None): lazylibrarian.MAG_UPDATE = 1 # noinspection PyBroadException try: myDB = database.DBConnection() onetitle = title if onetitle: mag_path = lazylibrarian.CONFIG['MAG_DEST_FOLDER'].replace('$Title', onetitle) else: mag_path = os.path.dirname(lazylibrarian.CONFIG['MAG_DEST_FOLDER']) if lazylibrarian.CONFIG['MAG_RELATIVE']: mag_path = os.path.join(lazylibrarian.DIRECTORY('eBook'), mag_path) if PY2: mag_path = mag_path.encode(lazylibrarian.SYS_ENCODING) if lazylibrarian.CONFIG['FULL_SCAN'] and not onetitle: mags = myDB.select('select * from Issues') # check all the issues are still there, delete entry if not for mag in mags: title = mag['Title'] issuedate = mag['IssueDate'] issuefile = mag['IssueFile'] if issuefile and not os.path.isfile(issuefile): myDB.action('DELETE from Issues where issuefile=?', (issuefile,)) logger.info('Issue %s - %s deleted as not found on disk' % (title, issuedate)) controlValueDict = {"Title": title} newValueDict = { "LastAcquired": None, # clear magazine dates "IssueDate": None, # we will fill them in again later "LatestCover": None, "IssueStatus": "Skipped" # assume there are no issues now } myDB.upsert("magazines", newValueDict, controlValueDict) logger.debug('Magazine %s details reset' % title) # now check the magazine titles and delete any with no issues if lazylibrarian.CONFIG['MAG_DELFOLDER']: mags = myDB.select('SELECT Title,count(Title) as counter from issues group by Title') for mag in mags: title = mag['Title'] issues = mag['counter'] if not issues: logger.debug('Magazine %s deleted as no issues found' % title) myDB.action('DELETE from magazines WHERE Title=?', (title,)) logger.info(' Checking [%s] for magazines' % mag_path) matchString = '' for char in lazylibrarian.CONFIG['MAG_DEST_FILE']: matchString = matchString + '\\' + char # massage the MAG_DEST_FILE config parameter into something we can use # with regular expression matching booktypes = '' count = -1 booktype_list = getList(lazylibrarian.CONFIG['MAG_TYPE']) for book_type in booktype_list: count += 1 if count == 0: booktypes = book_type else: booktypes = booktypes + '|' + book_type match = matchString.replace("\\$\\I\\s\\s\\u\\e\\D\\a\\t\\e", "(?P<issuedate>.*?)").replace( "\\$\\T\\i\\t\\l\\e", "(?P<title>.*?)") + '\.[' + booktypes + ']' title_pattern = re.compile(match, re.VERBOSE) match = matchString.replace("\\$\\I\\s\\s\\u\\e\\D\\a\\t\\e", "(?P<issuedate>.*?)").replace( "\\$\\T\\i\\t\\l\\e", "") + '\.[' + booktypes + ']' date_pattern = re.compile(match, re.VERBOSE) # try to ensure startdir is str as os.walk can fail if it tries to convert a subdir or file # to utf-8 and fails (eg scandinavian characters in ascii 8bit) for rootdir, dirnames, filenames in os.walk(makeBytestr(mag_path)): rootdir = makeUnicode(rootdir) filenames = [makeUnicode(item) for item in filenames] for fname in filenames: # maybe not all magazines will be pdf? if is_valid_booktype(fname, booktype='mag'): issuedate = '' # noinspection PyBroadException try: match = title_pattern.match(fname) if match: title = match.group("title") issuedate = match.group("issuedate") if lazylibrarian.LOGLEVEL & lazylibrarian.log_magdates: logger.debug("Title pattern [%s][%s]" % (title, issuedate)) match = True else: logger.debug("Title pattern match failed for [%s]" % fname) except Exception: match = False if not match: # noinspection PyBroadException try: match = date_pattern.match(fname) if match: issuedate = match.group("issuedate") title = os.path.basename(rootdir) if lazylibrarian.LOGLEVEL & lazylibrarian.log_magdates: logger.debug("Date pattern [%s][%s]" % (title, issuedate)) match = True else: logger.debug("Date pattern match failed for [%s]" % fname) except Exception: match = False if not match: title = os.path.basename(rootdir) issuedate = '' dic = {'.': ' ', '-': ' ', '/': ' ', '+': ' ', '_': ' ', '(': '', ')': '', '[': ' ', ']': ' ', '#': '# '} if issuedate: exploded = replace_all(issuedate, dic).split() regex_pass, issuedate, year = lazylibrarian.searchmag.get_issue_date(exploded) if lazylibrarian.LOGLEVEL & lazylibrarian.log_magdates: logger.debug("Date regex [%s][%s][%s]" % (regex_pass, issuedate, year)) if not regex_pass: issuedate = '' if not issuedate: exploded = replace_all(fname, dic).split() regex_pass, issuedate, year = lazylibrarian.searchmag.get_issue_date(exploded) if lazylibrarian.LOGLEVEL & lazylibrarian.log_magdates: logger.debug("File regex [%s][%s][%s]" % (regex_pass, issuedate, year)) if not regex_pass: issuedate = '' if not issuedate: logger.warn("Invalid name format for [%s]" % fname) continue issuefile = os.path.join(rootdir, fname) # full path to issue.pdf mtime = os.path.getmtime(issuefile) iss_acquired = datetime.date.isoformat(datetime.date.fromtimestamp(mtime)) if lazylibrarian.CONFIG['MAG_RENAME']: filedate = issuedate if issuedate and issuedate.isdigit(): if len(issuedate) == 8: if check_year(issuedate[:4]): filedate = 'Issue %d %s' % (int(issuedate[4:]), issuedate[:4]) else: filedate = 'Vol %d Iss %d' % (int(issuedate[:4]), int(issuedate[4:])) elif len(issuedate) == 12: filedate = 'Vol %d Iss %d %s' % (int(issuedate[4:8]), int(issuedate[8:]), issuedate[:4]) else: filedate = str(issuedate).zfill(4) extn = os.path.splitext(fname)[1] newfname = lazylibrarian.CONFIG['MAG_DEST_FILE'].replace('$Title', title).replace( '$IssueDate', filedate) newfname = newfname + extn if newfname and newfname != fname: logger.debug("Rename %s -> %s" % (fname, newfname)) newissuefile = os.path.join(rootdir, newfname) newissuefile = safe_move(issuefile, newissuefile) if os.path.exists(issuefile.replace(extn, '.jpg')): safe_move(issuefile.replace(extn, '.jpg'), newissuefile.replace(extn, '.jpg')) if os.path.exists(issuefile.replace(extn, '.opf')): safe_move(issuefile.replace(extn, '.opf'), newissuefile.replace(extn, '.opf')) issuefile = newissuefile logger.debug("Found %s Issue %s" % (title, issuedate)) controlValueDict = {"Title": title} # is this magazine already in the database? mag_entry = myDB.match( 'SELECT LastAcquired,IssueDate,MagazineAdded,CoverPage from magazines WHERE Title=?', (title,)) if not mag_entry: # need to add a new magazine to the database newValueDict = { "Reject": None, "Status": "Active", "MagazineAdded": None, "LastAcquired": None, "LatestCover": None, "IssueDate": None, "IssueStatus": "Skipped", "Regex": None, "CoverPage": 1 } logger.debug("Adding magazine %s" % title) myDB.upsert("magazines", newValueDict, controlValueDict) magissuedate = None magazineadded = None maglastacquired = None magcoverpage = 1 else: maglastacquired = mag_entry['LastAcquired'] magissuedate = mag_entry['IssueDate'] magazineadded = mag_entry['MagazineAdded'] magissuedate = str(magissuedate).zfill(4) magcoverpage = mag_entry['CoverPage'] issuedate = str(issuedate).zfill(4) # for sorting issue numbers # is this issue already in the database? issue_id = create_id("%s %s" % (title, issuedate)) iss_entry = myDB.match('SELECT Title,IssueFile from issues WHERE Title=? and IssueDate=?', (title, issuedate)) new_entry = False if not iss_entry or iss_entry['IssueFile'] != issuefile: new_entry = True # new entry or name changed if not iss_entry: logger.debug("Adding issue %s %s" % (title, issuedate)) else: logger.debug("Updating issue %s %s" % (title, issuedate)) controlValueDict = {"Title": title, "IssueDate": issuedate} newValueDict = { "IssueAcquired": iss_acquired, "IssueID": issue_id, "IssueFile": issuefile } myDB.upsert("Issues", newValueDict, controlValueDict) ignorefile = os.path.join(os.path.dirname(issuefile), '.ll_ignore') with open(ignorefile, 'a'): os.utime(ignorefile, None) createMagCover(issuefile, pagenum=magcoverpage, refresh=new_entry) lazylibrarian.postprocess.processMAGOPF(issuefile, title, issuedate, issue_id, overwrite=new_entry) # see if this issues date values are useful controlValueDict = {"Title": title} if not mag_entry: # new magazine, this is the only issue newValueDict = { "MagazineAdded": iss_acquired, "LastAcquired": iss_acquired, "LatestCover": os.path.splitext(issuefile)[0] + '.jpg', "IssueDate": issuedate, "IssueStatus": "Open" } myDB.upsert("magazines", newValueDict, controlValueDict) else: # Set magazine_issuedate to issuedate of most recent issue we have # Set latestcover to most recent issue cover # Set magazine_added to acquired date of earliest issue we have # Set magazine_lastacquired to acquired date of most recent issue we have # acquired dates are read from magazine file timestamps newValueDict = {"IssueStatus": "Open"} if not magazineadded or iss_acquired < magazineadded: newValueDict["MagazineAdded"] = iss_acquired if not maglastacquired or iss_acquired > maglastacquired: newValueDict["LastAcquired"] = iss_acquired if not magissuedate or issuedate >= magissuedate: newValueDict["IssueDate"] = issuedate newValueDict["LatestCover"] = os.path.splitext(issuefile)[0] + '.jpg' myDB.upsert("magazines", newValueDict, controlValueDict) if lazylibrarian.CONFIG['FULL_SCAN'] and not onetitle: magcount = myDB.match("select count(*) from magazines") isscount = myDB.match("select count(*) from issues") logger.info("Magazine scan complete, found %s magazine%s, %s issue%s" % (magcount['count(*)'], plural(magcount['count(*)']), isscount['count(*)'], plural(isscount['count(*)']))) else: logger.info("Magazine scan complete") lazylibrarian.MAG_UPDATE = 0 except Exception: lazylibrarian.MAG_UPDATE = 0 logger.error('Unhandled exception in magazineScan: %s' % traceback.format_exc())
def encode_multipart(fields, files, boundary=None): r"""Encode dict of form fields and dict of files as multipart/form-data. Return tuple of (body_string, headers_dict). Each value in files is a dict with required keys 'filename' and 'content', and optional 'mimetype' (if not specified, tries to guess mime type or uses 'application/octet-stream'). >>> body, headers = encode_multipart({'FIELD': 'VALUE'}, ... {'FILE': {'filename': 'F.TXT', 'content': 'CONTENT'}}, ... boundary='BOUNDARY') >>> print('\n'.join(repr(l) for l in body.split('\r\n'))) '--BOUNDARY' 'Content-Disposition: form-data; name="FIELD"' '' 'VALUE' '--BOUNDARY' 'Content-Disposition: form-data; name="FILE"; filename="F.TXT"' 'Content-Type: text/plain' '' 'CONTENT' '--BOUNDARY--' '' >>> print(sorted(headers.items())) [('Content-Length', '193'), ('Content-Type', 'multipart/form-data; boundary=BOUNDARY')] >>> len(body) 193 """ def escape_quote(s): s = makeUnicode(s) return s.replace('"', '\\"') if boundary is None: boundary = ''.join(random.choice(_BOUNDARY_CHARS) for _ in range(30)) lines = [] if fields: fields = dict((makeBytestr(k), makeBytestr(v)) for k, v in fields.items()) for name, value in list(fields.items()): lines.extend(( '--{0}'.format(boundary), 'Content-Disposition: form-data; name="{0}"'.format(escape_quote(name)), '', makeUnicode(value), )) if files: for name, value in list(files.items()): filename = value['filename'] if 'mimetype' in value: mimetype = value['mimetype'] else: mimetype = mimetypes.guess_type(filename)[0] or 'application/octet-stream' lines.extend(( '--{0}'.format(boundary), 'Content-Disposition: form-data; name="{0}"; filename="{1}"'.format( escape_quote(name), escape_quote(filename)), 'Content-Type: {0}'.format(mimetype), '', value['content'], )) lines.extend(( '--{0}--'.format(boundary), '', )) body = makeBytestr('\r\n'.join(lines)) headers = { 'Content-Type': 'multipart/form-data; boundary={0}'.format(boundary), 'Content-Length': str(len(body)), } return body, headers
def magazineScan(): lazylibrarian.MAG_UPDATE = 1 # noinspection PyBroadException try: myDB = database.DBConnection() mag_path = lazylibrarian.CONFIG['MAG_DEST_FOLDER'] mag_path = mag_path.split('$')[0] if lazylibrarian.CONFIG['MAG_RELATIVE']: if mag_path[0] not in '._': mag_path = '_' + mag_path mag_path = os.path.join(lazylibrarian.DIRECTORY('eBook'), mag_path) mag_path = mag_path.encode(lazylibrarian.SYS_ENCODING) if lazylibrarian.CONFIG['FULL_SCAN']: mags = myDB.select('select * from Issues') # check all the issues are still there, delete entry if not for mag in mags: title = mag['Title'] issuedate = mag['IssueDate'] issuefile = mag['IssueFile'] if issuefile and not os.path.isfile(issuefile): myDB.action('DELETE from Issues where issuefile=?', (issuefile, )) logger.info('Issue %s - %s deleted as not found on disk' % (title, issuedate)) controlValueDict = {"Title": title} newValueDict = { "LastAcquired": None, # clear magazine dates "IssueDate": None, # we will fill them in again later "LatestCover": None, "IssueStatus": "Skipped" # assume there are no issues now } myDB.upsert("magazines", newValueDict, controlValueDict) logger.debug('Magazine %s details reset' % title) mags = myDB.select('SELECT * from magazines') # now check the magazine titles and delete any with no issues for mag in mags: title = mag['Title'] count = myDB.select( 'SELECT COUNT(Title) as counter FROM issues WHERE Title=?', (title, )) issues = count[0]['counter'] if not issues: logger.debug('Magazine %s deleted as no issues found' % title) myDB.action('DELETE from magazines WHERE Title=?', (title, )) logger.info(' Checking [%s] for magazines' % mag_path) matchString = '' for char in lazylibrarian.CONFIG['MAG_DEST_FILE']: matchString = matchString + '\\' + char # massage the MAG_DEST_FILE config parameter into something we can use # with regular expression matching booktypes = '' count = -1 booktype_list = getList(lazylibrarian.CONFIG['MAG_TYPE']) for book_type in booktype_list: count += 1 if count == 0: booktypes = book_type else: booktypes = booktypes + '|' + book_type match = matchString.replace( "\\$\\I\\s\\s\\u\\e\\D\\a\\t\\e", "(?P<issuedate>.*?)").replace( "\\$\\T\\i\\t\\l\\e", "(?P<title>.*?)") + '\.[' + booktypes + ']' title_pattern = re.compile(match, re.VERBOSE) match = matchString.replace( "\\$\\I\\s\\s\\u\\e\\D\\a\\t\\e", "(?P<issuedate>.*?)").replace( "\\$\\T\\i\\t\\l\\e", "") + '\.[' + booktypes + ']' date_pattern = re.compile(match, re.VERBOSE) # try to ensure startdir is str as os.walk can fail if it tries to convert a subdir or file # to utf-8 and fails (eg scandinavian characters in ascii 8bit) for rootdir, dirnames, filenames in os.walk(makeBytestr(mag_path)): rootdir = makeUnicode(rootdir) filenames = [makeUnicode(item) for item in filenames] for fname in filenames: # maybe not all magazines will be pdf? if is_valid_booktype(fname, booktype='mag'): issuedate = '' # noinspection PyBroadException try: match = title_pattern.match(fname) if match: issuedate = match.group("issuedate") title = match.group("title") match = True else: match = False except Exception: match = False if not match: try: match = date_pattern.match(fname) if match: issuedate = match.group("issuedate") title = os.path.basename(rootdir) else: logger.debug("Pattern match failed for [%s]" % fname) continue except Exception as e: logger.debug("Invalid name format for [%s] %s %s" % (fname, type(e).__name__, str(e))) continue logger.debug("Found %s Issue %s" % (title, fname)) issuefile = os.path.join(rootdir, fname) # full path to issue.pdf mtime = os.path.getmtime(issuefile) iss_acquired = datetime.date.isoformat( datetime.date.fromtimestamp(mtime)) controlValueDict = {"Title": title} # is this magazine already in the database? mag_entry = myDB.match( 'SELECT LastAcquired, IssueDate, MagazineAdded from magazines WHERE Title=?', (title, )) if not mag_entry: # need to add a new magazine to the database newValueDict = { "Reject": None, "Status": "Active", "MagazineAdded": None, "LastAcquired": None, "LatestCover": None, "IssueDate": None, "IssueStatus": "Skipped", "Regex": None } logger.debug("Adding magazine %s" % title) myDB.upsert("magazines", newValueDict, controlValueDict) magissuedate = None magazineadded = None else: maglastacquired = mag_entry['LastAcquired'] magissuedate = mag_entry['IssueDate'] magazineadded = mag_entry['MagazineAdded'] magissuedate = str(magissuedate).zfill(4) issuedate = str(issuedate).zfill( 4) # for sorting issue numbers # is this issue already in the database? controlValueDict = {"Title": title, "IssueDate": issuedate} issue_id = create_id("%s %s" % (title, issuedate)) iss_entry = myDB.match( 'SELECT Title from issues WHERE Title=? and IssueDate=?', (title, issuedate)) if not iss_entry: newValueDict = { "IssueAcquired": iss_acquired, "IssueID": issue_id, "IssueFile": issuefile } myDB.upsert("Issues", newValueDict, controlValueDict) logger.debug("Adding issue %s %s" % (title, issuedate)) create_cover(issuefile) lazylibrarian.postprocess.processMAGOPF( issuefile, title, issuedate, issue_id) # see if this issues date values are useful controlValueDict = {"Title": title} if not mag_entry: # new magazine, this is the only issue newValueDict = { "MagazineAdded": iss_acquired, "LastAcquired": iss_acquired, "LatestCover": os.path.splitext(issuefile)[0] + '.jpg', "IssueDate": issuedate, "IssueStatus": "Open" } myDB.upsert("magazines", newValueDict, controlValueDict) else: # Set magazine_issuedate to issuedate of most recent issue we have # Set latestcover to most recent issue cover # Set magazine_added to acquired date of earliest issue we have # Set magazine_lastacquired to acquired date of most recent issue we have # acquired dates are read from magazine file timestamps newValueDict = {"IssueStatus": "Open"} if not magazineadded or iss_acquired < magazineadded: newValueDict["MagazineAdded"] = iss_acquired if not maglastacquired or iss_acquired > maglastacquired: newValueDict["LastAcquired"] = iss_acquired if not magissuedate or issuedate >= magissuedate: newValueDict["IssueDate"] = issuedate newValueDict["LatestCover"] = os.path.splitext( issuefile)[0] + '.jpg' myDB.upsert("magazines", newValueDict, controlValueDict) magcount = myDB.match("select count(*) from magazines") isscount = myDB.match("select count(*) from issues") logger.info("Magazine scan complete, found %s magazine%s, %s issue%s" % (magcount['count(*)'], plural(magcount['count(*)']), isscount['count(*)'], plural(isscount['count(*)']))) lazylibrarian.MAG_UPDATE = 0 except Exception: lazylibrarian.MAG_UPDATE = 0 logger.error('Unhandled exception in magazineScan: %s' % traceback.format_exc())
def audioRename(bookid): for item in ['$Part', '$Title']: if item not in lazylibrarian.CONFIG['AUDIOBOOK_DEST_FILE']: logger.error("Unable to audioRename, check AUDIOBOOK_DEST_FILE") return '' myDB = database.DBConnection() cmd = 'select AuthorName,BookName,AudioFile from books,authors where books.AuthorID = authors.AuthorID and bookid=?' exists = myDB.match(cmd, (bookid,)) if exists: book_filename = exists['AudioFile'] if book_filename: r = os.path.dirname(book_filename) else: logger.debug("No filename for %s in audioRename %s" % bookid) return '' else: logger.debug("Invalid bookid in audioRename %s" % bookid) return '' if not TinyTag: logger.warn("TinyTag library not available") return '' cnt = 0 parts = [] author = '' book = '' total = 0 audio_file = '' for f in os.listdir(makeBytestr(r)): f = makeUnicode(f) if is_valid_booktype(f, booktype='audiobook'): cnt += 1 audio_file = f try: id3r = TinyTag.get(os.path.join(r, f)) performer = id3r.artist composer = id3r.composer book = id3r.album track = id3r.track total = id3r.track_total track = check_int(track, 0) total = check_int(total, 0) if composer: # if present, should be author author = composer elif performer: # author, or narrator if composer == author author = performer if author and book: parts.append([track, book, author, f]) except Exception as e: logger.error("tinytag %s %s" % (type(e).__name__, str(e))) pass logger.debug("%s found %s audiofile%s" % (exists['BookName'], cnt, plural(cnt))) if cnt == 1 and not parts: # single file audiobook parts = [1, exists['BookName'], exists['AuthorName'], audio_file] if cnt != len(parts): logger.warn("%s: Incorrect number of parts (found %i from %i)" % (exists['BookName'], len(parts), cnt)) return book_filename if total and total != cnt: logger.warn("%s: Reported %i parts, got %i" % (exists['BookName'], total, cnt)) return book_filename # check all parts have the same author and title if len(parts) > 1: for part in parts: if part[1] != book: logger.warn("%s: Inconsistent title: [%s][%s]" % (exists['BookName'], part[1], book)) return book_filename if part[2] != author: logger.warn("%s: Inconsistent author: [%s][%s]" % (exists['BookName'], part[2], author)) return book_filename # do we have any track info (value is 0 if not) if parts[0][0] == 0: tokmatch = '' # try to extract part information from filename. Search for token style of part 1 in this order... for token in [' 001.', ' 01.', ' 1.', ' 001 ', ' 01 ', ' 1 ', '01']: if tokmatch: break for part in parts: if token in part[3]: tokmatch = token break if tokmatch: # we know the numbering style, get numbers for the other parts cnt = 0 while cnt < len(parts): cnt += 1 if tokmatch == ' 001.': pattern = ' %s.' % str(cnt).zfill(3) elif tokmatch == ' 01.': pattern = ' %s.' % str(cnt).zfill(2) elif tokmatch == ' 1.': pattern = ' %s.' % str(cnt) elif tokmatch == ' 001 ': pattern = ' %s ' % str(cnt).zfill(3) elif tokmatch == ' 01 ': pattern = ' %s ' % str(cnt).zfill(2) elif tokmatch == ' 1 ': pattern = ' %s ' % str(cnt) else: pattern = '%s' % str(cnt).zfill(2) # standardise numbering of the parts for part in parts: if pattern in part[3]: part[0] = cnt break # check all parts are present cnt = 0 found = True while found and cnt < len(parts): found = False cnt += 1 for part in parts: trk = part[0] if trk == cnt: found = True break if not found: logger.warn("%s: No part %i found" % (exists['BookName'], cnt)) return book_filename # if we get here, looks like we have all the parts needed to rename properly seriesinfo = seriesInfo(bookid) dest_path = lazylibrarian.CONFIG['EBOOK_DEST_FOLDER'].replace( '$Author', author).replace( '$Title', book).replace( '$Series', seriesinfo['Full']).replace( '$SerName', seriesinfo['Name']).replace( '$SerNum', seriesinfo['Num']).replace( '$$', ' ') dest_path = ' '.join(dest_path.split()).strip() dest_path = replace_all(dest_path, __dic__) dest_dir = lazylibrarian.DIRECTORY('Audio') dest_path = os.path.join(dest_dir, dest_path) if r != dest_path: try: dest_path = safe_move(r, dest_path) r = dest_path except Exception as why: if not os.path.isdir(dest_path): logger.error('Unable to create directory %s: %s' % (dest_path, why)) for part in parts: pattern = lazylibrarian.CONFIG['AUDIOBOOK_DEST_FILE'] seriesinfo = seriesInfo(bookid) pattern = pattern.replace( '$Author', author).replace( '$Title', book).replace( '$Part', str(part[0]).zfill(len(str(len(parts))))).replace( '$Total', str(len(parts))).replace( '$Series', seriesinfo['Full']).replace( '$SerName', seriesinfo['Name']).replace( '$SerNum', seriesinfo['Num']).replace( '$$', ' ') pattern = ' '.join(pattern.split()).strip() n = os.path.join(r, pattern + os.path.splitext(part[3])[1]) o = os.path.join(r, part[3]) if o != n: try: n = safe_move(o, n) if part[0] == 1: book_filename = n # return part 1 of set logger.debug('%s: audioRename [%s] to [%s]' % (exists['BookName'], o, n)) except Exception as e: logger.error('Unable to rename [%s] to [%s] %s %s' % (o, n, type(e).__name__, str(e))) return book_filename
def get_cached_request(url, useCache=True, cache="XML"): # hashfilename = hash of url # if hashfilename exists in cache and isn't too old, return its contents # if not, read url and store the result in the cache # return the result, and boolean True if source was cache # cacheLocation = cache + "Cache" cacheLocation = os.path.join(lazylibrarian.CACHEDIR, cacheLocation) if not os.path.exists(cacheLocation): os.mkdir(cacheLocation) myhash = md5_utf8(url) valid_cache = False source = None hashfilename = cacheLocation + os.path.sep + myhash + "." + cache.lower() expiry = lazylibrarian.CONFIG['CACHE_AGE'] * 24 * 60 * 60 # expire cache after this many seconds if useCache and os.path.isfile(hashfilename): cache_modified_time = os.stat(hashfilename).st_mtime time_now = time.time() if cache_modified_time < time_now - expiry: # Cache entry is too old, delete it if lazylibrarian.LOGLEVEL & lazylibrarian.log_cache: logger.debug("Expiring %s" % myhash) os.remove(hashfilename) else: valid_cache = True if valid_cache: lazylibrarian.CACHE_HIT = int(lazylibrarian.CACHE_HIT) + 1 if lazylibrarian.LOGLEVEL & lazylibrarian.log_cache: logger.debug("CacheHandler: Returning CACHED response %s for %s" % (hashfilename, url)) if cache == "JSON": try: source = json.load(open(hashfilename)) except ValueError: logger.error("Error decoding json from %s" % hashfilename) return None, False elif cache == "XML": with open(hashfilename, "rb") as cachefile: result = cachefile.read() if result and result.startswith(b'<?xml'): try: source = ElementTree.fromstring(result) except UnicodeEncodeError: # seems sometimes the page contains utf-16 but the header says it's utf-8 try: result = result.decode('utf-16').encode('utf-8') source = ElementTree.fromstring(result) except (ElementTree.ParseError, UnicodeEncodeError, UnicodeDecodeError): logger.error("Error parsing xml from %s" % hashfilename) source = None except ElementTree.ParseError: logger.error("Error parsing xml from %s" % hashfilename) source = None if source is None: logger.error("Error reading xml from %s" % hashfilename) os.remove(hashfilename) return None, False else: lazylibrarian.CACHE_MISS = int(lazylibrarian.CACHE_MISS) + 1 if cache == 'XML': gr_api_sleep() result, success = fetchURL(url, raw=True) else: result, success = fetchURL(url) if success: if lazylibrarian.LOGLEVEL & lazylibrarian.log_cache: logger.debug("CacheHandler: Storing %s %s for %s" % (cache, myhash, url)) if cache == "JSON": try: source = json.loads(result) if not expiry: return source, False except Exception as e: logger.error("%s decoding json from %s" % (type(e).__name__, url)) logger.debug("%s : %s" % (e, result)) return None, False json.dump(source, open(hashfilename, "w")) elif cache == "XML": result = makeBytestr(result) if result and result.startswith(b'<?xml'): try: source = ElementTree.fromstring(result) if not expiry: return source, False except UnicodeEncodeError: # sometimes we get utf-16 data labelled as utf-8 try: result = result.decode('utf-16').encode('utf-8') source = ElementTree.fromstring(result) if not expiry: return source, False except (ElementTree.ParseError, UnicodeEncodeError, UnicodeDecodeError): logger.error("Error parsing xml from %s" % url) source = None except ElementTree.ParseError: logger.error("Error parsing xml from %s" % url) source = None if source is not None: with open(hashfilename, "wb") as cachefile: cachefile.write(result) else: logger.error("Error getting xml data from %s" % url) return None, False else: logger.debug("Got error response for %s: %s" % (url, result.split('<')[0])) if 'goodreads' in url and '503' in result: time.sleep(1) return None, False return source, valid_cache