def _get_metadata(self, path): from calibre.ebooks.metadata.epub import get_metadata self._log_location("_get_metadata: path=%s" % path) with open(path, 'rb') as f: mi = get_metadata(f) self._log_location("Finish!!!!") return mi
def test_get(self): # {{{ 'Test /get' with self.create_server() as server: db = server.handler.router.ctx.library_broker.get(None) conn = server.connect() def get(what, book_id, library_id=None, q=''): q = ('?' + q) if q else q conn.request('GET', '/get/%s/%s' % (what, book_id) + (('/' + library_id) if library_id else '') + q) r = conn.getresponse() return r, r.read() # Test various invalid parameters def bad(*args): r, data = get(*args) self.ae(r.status, httplib.NOT_FOUND) bad('xxx', 1) bad('fmt1', 10) bad('fmt1', 1, 'zzzz') bad('fmt1', 'xx') # Test simple fetching of format without metadata update r, data = get('fmt1', 1, db.server_library_id) self.ae(data, db.format(1, 'fmt1')) self.assertIsNotNone(r.getheader('Content-Disposition')) self.ae(r.getheader('Used-Cache'), 'no') r, data = get('fmt1', 1) self.ae(data, db.format(1, 'fmt1')) self.ae(r.getheader('Used-Cache'), 'yes') # Test fetching of format with metadata update raw = P('quick_start/eng.epub', data=True) r, data = get('epub', 1) self.ae(r.status, httplib.OK) etag = r.getheader('ETag') self.assertIsNotNone(etag) self.ae(r.getheader('Used-Cache'), 'no') self.assertTrue(data.startswith(b'PK')) self.assertGreaterEqual(len(data), len(raw)) db.set_field('title', {1:'changed'}) r, data = get('epub', 1) self.assertNotEqual(r.getheader('ETag'), etag) etag = r.getheader('ETag') self.ae(r.getheader('Used-Cache'), 'no') mi = get_metadata(BytesIO(data), extract_cover=False) self.ae(mi.title, 'changed') r, data = get('epub', 1) self.ae(r.getheader('Used-Cache'), 'yes') # Test plugboards import calibre.library.save_to_disk as c orig, c.DEBUG = c.DEBUG, False try: db.set_pref('plugboards', {u'epub': {u'content_server': [[u'changed, {title}', u'title']]}}) # this is needed as the cache is not invalidated for plugboard changes db.set_field('title', {1:'again'}) r, data = get('epub', 1) self.assertNotEqual(r.getheader('ETag'), etag) etag = r.getheader('ETag') self.ae(r.getheader('Used-Cache'), 'no') mi = get_metadata(BytesIO(data), extract_cover=False) self.ae(mi.title, 'changed, again') finally: c.DEBUG = orig # Test the serving of covers def change_cover(count, book_id=2): cpath = db.format_abspath(book_id, '__COVER_INTERNAL__') db.set_cover({2:I('lt.png', data=True)}) t = time.time() + 1 + count # Ensure mtime changes, needed on OS X where HFS+ has a 1s # mtime resolution os.utime(cpath, (t, t)) r, data = get('cover', 1) self.ae(r.status, httplib.OK) self.ae(data, db.cover(1)) self.ae(r.getheader('Used-Cache'), 'no') self.ae(r.getheader('Content-Type'), 'image/jpeg') r, data = get('cover', 1) self.ae(r.status, httplib.OK) self.ae(data, db.cover(1)) self.ae(r.getheader('Used-Cache'), 'yes') r, data = get('cover', 3) self.ae(r.status, httplib.OK) # Auto generated cover r, data = get('thumb', 1) self.ae(r.status, httplib.OK) self.ae(identify(data), ('jpeg', 60, 60)) self.ae(r.getheader('Used-Cache'), 'no') r, data = get('thumb', 1) self.ae(r.status, httplib.OK) self.ae(r.getheader('Used-Cache'), 'yes') r, data = get('thumb', 1, q='sz=100') self.ae(r.status, httplib.OK) self.ae(identify(data), ('jpeg', 100, 100)) self.ae(r.getheader('Used-Cache'), 'no') r, data = get('thumb', 1, q='sz=100x100') self.ae(r.status, httplib.OK) self.ae(r.getheader('Used-Cache'), 'yes') change_cover(1, 1) r, data = get('thumb', 1, q='sz=100') self.ae(r.status, httplib.OK) self.ae(identify(data), ('jpeg', 100, 100)) self.ae(r.getheader('Used-Cache'), 'no') # Test file sharing in cache r, data = get('cover', 2) self.ae(r.status, httplib.OK) self.ae(data, db.cover(2)) self.ae(r.getheader('Used-Cache'), 'no') path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8') f, fdata = share_open(path, 'rb'), data # Now force an update change_cover(1) r, data = get('cover', 2) self.ae(r.status, httplib.OK) self.ae(data, db.cover(2)) self.ae(r.getheader('Used-Cache'), 'no') path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8') f2, f2data = share_open(path, 'rb'), data # Do it again change_cover(2) r, data = get('cover', 2) self.ae(r.status, httplib.OK) self.ae(data, db.cover(2)) self.ae(r.getheader('Used-Cache'), 'no') self.ae(f.read(), fdata) self.ae(f2.read(), f2data) # Test serving of metadata as opf r, data = get('opf', 1) self.ae(r.status, httplib.OK) self.ae(r.getheader('Content-Type'), 'application/oebps-package+xml; charset=UTF-8') self.assertIsNotNone(r.getheader('Last-Modified')) opf = OPF(BytesIO(data), populate_spine=False, try_to_guess_cover=False) self.ae(db.field_for('title', 1), opf.title) self.ae(db.field_for('authors', 1), tuple(opf.authors)) conn.request('GET', '/get/opf/1', headers={'Accept-Encoding':'gzip'}) r = conn.getresponse() self.ae(r.status, httplib.OK), self.ae(r.getheader('Content-Encoding'), 'gzip') raw = r.read() self.ae(zlib.decompress(raw, 16+zlib.MAX_WBITS), data) # Test serving metadata as json r, data = get('json', 1) self.ae(r.status, httplib.OK) self.ae(db.field_for('title', 1), json.loads(data)['title']) conn.request('GET', '/get/json/1', headers={'Accept-Encoding':'gzip'}) r = conn.getresponse() self.ae(r.status, httplib.OK), self.ae(r.getheader('Content-Encoding'), 'gzip') raw = r.read() self.ae(zlib.decompress(raw, 16+zlib.MAX_WBITS), data)
def test_get(self): # {{{ 'Test /get' with self.create_server() as server: db = server.handler.router.ctx.library_broker.get(None) conn = server.connect() def get(what, book_id, library_id=None, q=''): q = ('?' + q) if q else q conn.request( 'GET', '/get/%s/%s' % (what, book_id) + (('/' + library_id) if library_id else '') + q) r = conn.getresponse() return r, r.read() # Test various invalid parameters def bad(*args): r, data = get(*args) self.ae(r.status, http_client.NOT_FOUND) bad('xxx', 1) bad('fmt1', 10) bad('fmt1', 1, 'zzzz') bad('fmt1', 'xx') # Test simple fetching of format without metadata update r, data = get('fmt1', 1, db.server_library_id) self.ae(data, db.format(1, 'fmt1')) self.assertIsNotNone(r.getheader('Content-Disposition')) self.ae(r.getheader('Used-Cache'), 'no') r, data = get('fmt1', 1) self.ae(data, db.format(1, 'fmt1')) self.ae(r.getheader('Used-Cache'), 'yes') # Test fetching of format with metadata update raw = P('quick_start/eng.epub', data=True) r, data = get('epub', 1) self.ae(r.status, http_client.OK) etag = r.getheader('ETag') self.assertIsNotNone(etag) self.ae(r.getheader('Used-Cache'), 'no') self.assertTrue(data.startswith(b'PK')) self.assertGreaterEqual(len(data), len(raw)) db.set_field('title', {1: 'changed'}) r, data = get('epub', 1) self.assertNotEqual(r.getheader('ETag'), etag) etag = r.getheader('ETag') self.ae(r.getheader('Used-Cache'), 'no') mi = get_metadata(BytesIO(data), extract_cover=False) self.ae(mi.title, 'changed') r, data = get('epub', 1) self.ae(r.getheader('Used-Cache'), 'yes') # Test plugboards import calibre.library.save_to_disk as c orig, c.DEBUG = c.DEBUG, False try: db.set_pref( 'plugboards', { u'epub': { u'content_server': [[u'changed, {title}', u'title']] } }) # this is needed as the cache is not invalidated for plugboard changes db.set_field('title', {1: 'again'}) r, data = get('epub', 1) self.assertNotEqual(r.getheader('ETag'), etag) etag = r.getheader('ETag') self.ae(r.getheader('Used-Cache'), 'no') mi = get_metadata(BytesIO(data), extract_cover=False) self.ae(mi.title, 'changed, again') finally: c.DEBUG = orig # Test the serving of covers def change_cover(count, book_id=2): cpath = db.format_abspath(book_id, '__COVER_INTERNAL__') db.set_cover({2: I('lt.png', data=True)}) t = time.time() + 1 + count # Ensure mtime changes, needed on OS X where HFS+ has a 1s # mtime resolution os.utime(cpath, (t, t)) r, data = get('cover', 1) self.ae(r.status, http_client.OK) self.ae(data, db.cover(1)) self.ae(r.getheader('Used-Cache'), 'no') self.ae(r.getheader('Content-Type'), 'image/jpeg') r, data = get('cover', 1) self.ae(r.status, http_client.OK) self.ae(data, db.cover(1)) self.ae(r.getheader('Used-Cache'), 'yes') r, data = get('cover', 3) self.ae(r.status, http_client.OK) # Auto generated cover r, data = get('thumb', 1) self.ae(r.status, http_client.OK) self.ae(identify(data), ('jpeg', 60, 60)) self.ae(r.getheader('Used-Cache'), 'no') r, data = get('thumb', 1) self.ae(r.status, http_client.OK) self.ae(r.getheader('Used-Cache'), 'yes') r, data = get('thumb', 1, q='sz=100') self.ae(r.status, http_client.OK) self.ae(identify(data), ('jpeg', 100, 100)) self.ae(r.getheader('Used-Cache'), 'no') r, data = get('thumb', 1, q='sz=100x100') self.ae(r.status, http_client.OK) self.ae(r.getheader('Used-Cache'), 'yes') change_cover(1, 1) r, data = get('thumb', 1, q='sz=100') self.ae(r.status, http_client.OK) self.ae(identify(data), ('jpeg', 100, 100)) self.ae(r.getheader('Used-Cache'), 'no') # Test file sharing in cache r, data = get('cover', 2) self.ae(r.status, http_client.OK) self.ae(data, db.cover(2)) self.ae(r.getheader('Used-Cache'), 'no') path = from_hex_unicode(r.getheader('Tempfile')) f, fdata = share_open(path, 'rb'), data # Now force an update change_cover(1) r, data = get('cover', 2) self.ae(r.status, http_client.OK) self.ae(data, db.cover(2)) self.ae(r.getheader('Used-Cache'), 'no') path = from_hex_unicode(r.getheader('Tempfile')) f2, f2data = share_open(path, 'rb'), data # Do it again change_cover(2) r, data = get('cover', 2) self.ae(r.status, http_client.OK) self.ae(data, db.cover(2)) self.ae(r.getheader('Used-Cache'), 'no') self.ae(f.read(), fdata) self.ae(f2.read(), f2data) # Test serving of metadata as opf r, data = get('opf', 1) self.ae(r.status, http_client.OK) self.ae(r.getheader('Content-Type'), 'application/oebps-package+xml; charset=UTF-8') self.assertIsNotNone(r.getheader('Last-Modified')) opf = OPF(BytesIO(data), populate_spine=False, try_to_guess_cover=False) self.ae(db.field_for('title', 1), opf.title) self.ae(db.field_for('authors', 1), tuple(opf.authors)) conn.request('GET', '/get/opf/1', headers={'Accept-Encoding': 'gzip'}) r = conn.getresponse() self.ae(r.status, http_client.OK), self.ae(r.getheader('Content-Encoding'), 'gzip') raw = r.read() self.ae(zlib.decompress(raw, 16 + zlib.MAX_WBITS), data) # Test serving metadata as json r, data = get('json', 1) self.ae(r.status, http_client.OK) self.ae(db.field_for('title', 1), json.loads(data)['title']) conn.request('GET', '/get/json/1', headers={'Accept-Encoding': 'gzip'}) r = conn.getresponse() self.ae(r.status, http_client.OK), self.ae(r.getheader('Content-Encoding'), 'gzip') raw = r.read() self.ae(zlib.decompress(raw, 16 + zlib.MAX_WBITS), data)
def _generate_thumbnail(self, book): ''' Fetch the cover image, generate a thumbnail, cache Extracts covers from zipped epubs ''' self._log_location(book.title) #self._log("book_path: %s" % book.path) #self._log("book: '%s' by %s uuid: %s" % (book.title, book.author, book.uuid)) # Parse the cover from the connected device, model Fetch_Annotations:_get_epub_toc() thumb_data = None thumb_path = book.path.rpartition('.')[0] + '.jpg' # Try getting the cover from the cache try: zfr = ZipFile(self.archive_path) thumb_data = zfr.read(thumb_path) if thumb_data == 'None': self._log("returning None from cover cache") zfr.close() return None except: self._log("opening cover cache for appending") zfw = ZipFile(self.archive_path, mode='a') else: self._log("returning thumb from cover cache") return thumb_data # Get the cover from the book try: stream = cStringIO.StringIO(self.ios.read(book.path, mode='rb')) mi = get_metadata(stream) if mi.cover_data is not None: img_data = cStringIO.StringIO(mi.cover_data[1]) except: if self.verbose: self._log("ERROR: unable to get cover from '%s'" % book.title) import traceback #traceback.print_exc() exc_type, exc_value, exc_traceback = sys.exc_info() self._log(traceback.format_exception_only(exc_type, exc_value)[0].strip()) return thumb_data # Generate a thumb try: im = PILImage.open(img_data) scaled, width, height = fit_image(im.size[0], im.size[1], 60, 80) im = im.resize((int(width), int(height)), PILImage.ANTIALIAS) thumb = cStringIO.StringIO() im.convert('RGB').save(thumb, 'JPEG') thumb_data = thumb.getvalue() thumb.close() self._log("SUCCESS: generated thumb for '%s', caching" % book.title) # Cache the tagged thumb zfw.writestr(thumb_path, thumb_data) except: if self.verbose: self._log("ERROR generating thumb for '%s', caching empty marker" % book.title) import traceback exc_type, exc_value, exc_traceback = sys.exc_info() self._log(traceback.format_exception_only(exc_type, exc_value)[0].strip()) # Cache the empty cover zfw.writestr(thumb_path, 'None') finally: img_data.close() zfw.close() return thumb_data
def test_get(self): # {{{ 'Test /get' with self.create_server() as server: db = server.handler.router.ctx.get_library() conn = server.connect() def get(what, book_id, library_id=None, q=''): q = ('?' + q) if q else q conn.request('GET', '/get/%s/%s' % (what, book_id) + (('/' + library_id) if library_id else '') + q) r = conn.getresponse() return r, r.read() # Test various invalid parameters def bad(*args): r, data = get(*args) self.ae(r.status, httplib.NOT_FOUND) bad('xxx', 1) bad('fmt1', 10) bad('fmt1', 1, 'zzzz') bad('fmt1', 'xx') # Test simple fetching of format without metadata update r, data = get('fmt1', 1, db.server_library_id) self.ae(data, db.format(1, 'fmt1')) self.assertIsNotNone(r.getheader('Content-Disposition')) self.ae(r.getheader('Used-Cache'), 'no') r, data = get('fmt1', 1) self.ae(data, db.format(1, 'fmt1')) self.ae(r.getheader('Used-Cache'), 'yes') # Test fetching of format with metadata update raw = P('quick_start/eng.epub', data=True) r, data = get('epub', 1) self.ae(r.status, httplib.OK) etag = r.getheader('ETag') self.assertIsNotNone(etag) self.ae(r.getheader('Used-Cache'), 'no') self.assertTrue(data.startswith(b'PK')) self.assertGreaterEqual(len(data), len(raw)) db.set_field('title', {1:'changed'}) r, data = get('epub', 1) self.assertNotEqual(r.getheader('ETag'), etag) etag = r.getheader('ETag') self.ae(r.getheader('Used-Cache'), 'no') mi = get_metadata(BytesIO(data), extract_cover=False) self.ae(mi.title, 'changed') r, data = get('epub', 1) self.ae(r.getheader('Used-Cache'), 'yes') # Test plugboards import calibre.library.save_to_disk as c orig, c.DEBUG = c.DEBUG, False try: db.set_pref('plugboards', {u'epub': {u'content_server': [[u'changed, {title}', u'title']]}}) # this is needed as the cache is not invalidated for plugboard changes db.set_field('title', {1:'again'}) r, data = get('epub', 1) self.assertNotEqual(r.getheader('ETag'), etag) etag = r.getheader('ETag') self.ae(r.getheader('Used-Cache'), 'no') mi = get_metadata(BytesIO(data), extract_cover=False) self.ae(mi.title, 'changed, again') finally: c.DEBUG = orig # Test the serving of covers r, data = get('cover', 1) self.ae(r.status, httplib.OK) self.ae(data, db.cover(1)) self.ae(r.getheader('Used-Cache'), 'no') self.ae(r.getheader('Content-Type'), 'image/jpeg') r, data = get('cover', 1) self.ae(r.status, httplib.OK) self.ae(data, db.cover(1)) self.ae(r.getheader('Used-Cache'), 'yes') r, data = get('cover', 3) self.ae(r.status, httplib.NOT_FOUND) r, data = get('thumb', 1) self.ae(r.status, httplib.OK) self.ae(identify_data(data), (60, 60, 'jpeg')) self.ae(r.getheader('Used-Cache'), 'no') r, data = get('thumb', 1) self.ae(r.status, httplib.OK) self.ae(r.getheader('Used-Cache'), 'yes') r, data = get('thumb', 1, q='sz=100') self.ae(r.status, httplib.OK) self.ae(identify_data(data), (100, 100, 'jpeg')) self.ae(r.getheader('Used-Cache'), 'no') r, data = get('thumb', 1, q='sz=100x100') self.ae(r.status, httplib.OK) self.ae(r.getheader('Used-Cache'), 'yes') db.set_cover({1:I('lt.png', data=True)}) r, data = get('thumb', 1, q='sz=100') self.ae(r.status, httplib.OK) self.ae(identify_data(data), (100, 100, 'jpeg')) self.ae(r.getheader('Used-Cache'), 'no') # Test serving of metadata as opf r, data = get('opf', 1) self.ae(r.status, httplib.OK) self.ae(r.getheader('Content-Type'), 'application/oebps-package+xml; charset=UTF-8') self.assertIsNotNone(r.getheader('Last-Modified')) opf = OPF(BytesIO(data), populate_spine=False, try_to_guess_cover=False) self.ae(db.field_for('title', 1), opf.title) self.ae(db.field_for('authors', 1), tuple(opf.authors)) conn.request('GET', '/get/opf/1', headers={'Accept-Encoding':'gzip'}) r = conn.getresponse() self.ae(r.status, httplib.OK), self.ae(r.getheader('Content-Encoding'), 'gzip') raw = r.read() self.ae(zlib.decompress(raw, 16+zlib.MAX_WBITS), data) # Test serving metadata as json r, data = get('json', 1) self.ae(r.status, httplib.OK) self.ae(db.field_for('title', 1), json.loads(data)['title']) conn.request('GET', '/get/json/1', headers={'Accept-Encoding':'gzip'}) r = conn.getresponse() self.ae(r.status, httplib.OK), self.ae(r.getheader('Content-Encoding'), 'gzip') raw = r.read() self.ae(zlib.decompress(raw, 16+zlib.MAX_WBITS), data)