def _get_metadata(self, path):
from calibre.ebooks.metadata.epub import get_metadata
self._log_location("_get_metadata: path=%s" % path)
with open(path, 'rb') as f:
mi = get_metadata(f)
self._log_location("Finish!!!!")
return mi
def _get_metadata(self, path):
from calibre.ebooks.metadata.epub import get_metadata
self._log_location("_get_metadata: path=%s" % path)
with open(path, 'rb') as f:
mi = get_metadata(f)
self._log_location("Finish!!!!")
return mi
def test_get(self): # {{{
'Test /get'
with self.create_server() as server:
db = server.handler.router.ctx.library_broker.get(None)
conn = server.connect()
def get(what, book_id, library_id=None, q=''):
q = ('?' + q) if q else q
conn.request('GET', '/get/%s/%s' % (what, book_id) + (('/' + library_id) if library_id else '') + q)
r = conn.getresponse()
return r, r.read()
# Test various invalid parameters
def bad(*args):
r, data = get(*args)
self.ae(r.status, httplib.NOT_FOUND)
bad('xxx', 1)
bad('fmt1', 10)
bad('fmt1', 1, 'zzzz')
bad('fmt1', 'xx')
# Test simple fetching of format without metadata update
r, data = get('fmt1', 1, db.server_library_id)
self.ae(data, db.format(1, 'fmt1'))
self.assertIsNotNone(r.getheader('Content-Disposition'))
self.ae(r.getheader('Used-Cache'), 'no')
r, data = get('fmt1', 1)
self.ae(data, db.format(1, 'fmt1'))
self.ae(r.getheader('Used-Cache'), 'yes')
# Test fetching of format with metadata update
raw = P('quick_start/eng.epub', data=True)
r, data = get('epub', 1)
self.ae(r.status, httplib.OK)
etag = r.getheader('ETag')
self.assertIsNotNone(etag)
self.ae(r.getheader('Used-Cache'), 'no')
self.assertTrue(data.startswith(b'PK'))
self.assertGreaterEqual(len(data), len(raw))
db.set_field('title', {1:'changed'})
r, data = get('epub', 1)
self.assertNotEqual(r.getheader('ETag'), etag)
etag = r.getheader('ETag')
self.ae(r.getheader('Used-Cache'), 'no')
mi = get_metadata(BytesIO(data), extract_cover=False)
self.ae(mi.title, 'changed')
r, data = get('epub', 1)
self.ae(r.getheader('Used-Cache'), 'yes')
# Test plugboards
import calibre.library.save_to_disk as c
orig, c.DEBUG = c.DEBUG, False
try:
db.set_pref('plugboards', {u'epub': {u'content_server': [[u'changed, {title}', u'title']]}})
# this is needed as the cache is not invalidated for plugboard changes
db.set_field('title', {1:'again'})
r, data = get('epub', 1)
self.assertNotEqual(r.getheader('ETag'), etag)
etag = r.getheader('ETag')
self.ae(r.getheader('Used-Cache'), 'no')
mi = get_metadata(BytesIO(data), extract_cover=False)
self.ae(mi.title, 'changed, again')
finally:
c.DEBUG = orig
# Test the serving of covers
def change_cover(count, book_id=2):
cpath = db.format_abspath(book_id, '__COVER_INTERNAL__')
db.set_cover({2:I('lt.png', data=True)})
t = time.time() + 1 + count
# Ensure mtime changes, needed on OS X where HFS+ has a 1s
# mtime resolution
os.utime(cpath, (t, t))
r, data = get('cover', 1)
self.ae(r.status, httplib.OK)
self.ae(data, db.cover(1))
self.ae(r.getheader('Used-Cache'), 'no')
self.ae(r.getheader('Content-Type'), 'image/jpeg')
r, data = get('cover', 1)
self.ae(r.status, httplib.OK)
self.ae(data, db.cover(1))
self.ae(r.getheader('Used-Cache'), 'yes')
r, data = get('cover', 3)
self.ae(r.status, httplib.OK) # Auto generated cover
r, data = get('thumb', 1)
self.ae(r.status, httplib.OK)
self.ae(identify(data), ('jpeg', 60, 60))
self.ae(r.getheader('Used-Cache'), 'no')
r, data = get('thumb', 1)
self.ae(r.status, httplib.OK)
self.ae(r.getheader('Used-Cache'), 'yes')
r, data = get('thumb', 1, q='sz=100')
self.ae(r.status, httplib.OK)
self.ae(identify(data), ('jpeg', 100, 100))
self.ae(r.getheader('Used-Cache'), 'no')
r, data = get('thumb', 1, q='sz=100x100')
self.ae(r.status, httplib.OK)
self.ae(r.getheader('Used-Cache'), 'yes')
change_cover(1, 1)
r, data = get('thumb', 1, q='sz=100')
self.ae(r.status, httplib.OK)
self.ae(identify(data), ('jpeg', 100, 100))
self.ae(r.getheader('Used-Cache'), 'no')
# Test file sharing in cache
r, data = get('cover', 2)
self.ae(r.status, httplib.OK)
self.ae(data, db.cover(2))
self.ae(r.getheader('Used-Cache'), 'no')
path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8')
f, fdata = share_open(path, 'rb'), data
# Now force an update
change_cover(1)
r, data = get('cover', 2)
self.ae(r.status, httplib.OK)
self.ae(data, db.cover(2))
self.ae(r.getheader('Used-Cache'), 'no')
path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8')
f2, f2data = share_open(path, 'rb'), data
# Do it again
change_cover(2)
r, data = get('cover', 2)
self.ae(r.status, httplib.OK)
self.ae(data, db.cover(2))
self.ae(r.getheader('Used-Cache'), 'no')
self.ae(f.read(), fdata)
self.ae(f2.read(), f2data)
# Test serving of metadata as opf
r, data = get('opf', 1)
self.ae(r.status, httplib.OK)
self.ae(r.getheader('Content-Type'), 'application/oebps-package+xml; charset=UTF-8')
self.assertIsNotNone(r.getheader('Last-Modified'))
opf = OPF(BytesIO(data), populate_spine=False, try_to_guess_cover=False)
self.ae(db.field_for('title', 1), opf.title)
self.ae(db.field_for('authors', 1), tuple(opf.authors))
conn.request('GET', '/get/opf/1', headers={'Accept-Encoding':'gzip'})
r = conn.getresponse()
self.ae(r.status, httplib.OK), self.ae(r.getheader('Content-Encoding'), 'gzip')
raw = r.read()
self.ae(zlib.decompress(raw, 16+zlib.MAX_WBITS), data)
# Test serving metadata as json
r, data = get('json', 1)
self.ae(r.status, httplib.OK)
self.ae(db.field_for('title', 1), json.loads(data)['title'])
conn.request('GET', '/get/json/1', headers={'Accept-Encoding':'gzip'})
r = conn.getresponse()
self.ae(r.status, httplib.OK), self.ae(r.getheader('Content-Encoding'), 'gzip')
raw = r.read()
self.ae(zlib.decompress(raw, 16+zlib.MAX_WBITS), data)
def test_get(self): # {{{
'Test /get'
with self.create_server() as server:
db = server.handler.router.ctx.library_broker.get(None)
conn = server.connect()
def get(what, book_id, library_id=None, q=''):
q = ('?' + q) if q else q
conn.request(
'GET', '/get/%s/%s' % (what, book_id) +
(('/' + library_id) if library_id else '') + q)
r = conn.getresponse()
return r, r.read()
# Test various invalid parameters
def bad(*args):
r, data = get(*args)
self.ae(r.status, http_client.NOT_FOUND)
bad('xxx', 1)
bad('fmt1', 10)
bad('fmt1', 1, 'zzzz')
bad('fmt1', 'xx')
# Test simple fetching of format without metadata update
r, data = get('fmt1', 1, db.server_library_id)
self.ae(data, db.format(1, 'fmt1'))
self.assertIsNotNone(r.getheader('Content-Disposition'))
self.ae(r.getheader('Used-Cache'), 'no')
r, data = get('fmt1', 1)
self.ae(data, db.format(1, 'fmt1'))
self.ae(r.getheader('Used-Cache'), 'yes')
# Test fetching of format with metadata update
raw = P('quick_start/eng.epub', data=True)
r, data = get('epub', 1)
self.ae(r.status, http_client.OK)
etag = r.getheader('ETag')
self.assertIsNotNone(etag)
self.ae(r.getheader('Used-Cache'), 'no')
self.assertTrue(data.startswith(b'PK'))
self.assertGreaterEqual(len(data), len(raw))
db.set_field('title', {1: 'changed'})
r, data = get('epub', 1)
self.assertNotEqual(r.getheader('ETag'), etag)
etag = r.getheader('ETag')
self.ae(r.getheader('Used-Cache'), 'no')
mi = get_metadata(BytesIO(data), extract_cover=False)
self.ae(mi.title, 'changed')
r, data = get('epub', 1)
self.ae(r.getheader('Used-Cache'), 'yes')
# Test plugboards
import calibre.library.save_to_disk as c
orig, c.DEBUG = c.DEBUG, False
try:
db.set_pref(
'plugboards', {
u'epub': {
u'content_server':
[[u'changed, {title}', u'title']]
}
})
# this is needed as the cache is not invalidated for plugboard changes
db.set_field('title', {1: 'again'})
r, data = get('epub', 1)
self.assertNotEqual(r.getheader('ETag'), etag)
etag = r.getheader('ETag')
self.ae(r.getheader('Used-Cache'), 'no')
mi = get_metadata(BytesIO(data), extract_cover=False)
self.ae(mi.title, 'changed, again')
finally:
c.DEBUG = orig
# Test the serving of covers
def change_cover(count, book_id=2):
cpath = db.format_abspath(book_id, '__COVER_INTERNAL__')
db.set_cover({2: I('lt.png', data=True)})
t = time.time() + 1 + count
# Ensure mtime changes, needed on OS X where HFS+ has a 1s
# mtime resolution
os.utime(cpath, (t, t))
r, data = get('cover', 1)
self.ae(r.status, http_client.OK)
self.ae(data, db.cover(1))
self.ae(r.getheader('Used-Cache'), 'no')
self.ae(r.getheader('Content-Type'), 'image/jpeg')
r, data = get('cover', 1)
self.ae(r.status, http_client.OK)
self.ae(data, db.cover(1))
self.ae(r.getheader('Used-Cache'), 'yes')
r, data = get('cover', 3)
self.ae(r.status, http_client.OK) # Auto generated cover
r, data = get('thumb', 1)
self.ae(r.status, http_client.OK)
self.ae(identify(data), ('jpeg', 60, 60))
self.ae(r.getheader('Used-Cache'), 'no')
r, data = get('thumb', 1)
self.ae(r.status, http_client.OK)
self.ae(r.getheader('Used-Cache'), 'yes')
r, data = get('thumb', 1, q='sz=100')
self.ae(r.status, http_client.OK)
self.ae(identify(data), ('jpeg', 100, 100))
self.ae(r.getheader('Used-Cache'), 'no')
r, data = get('thumb', 1, q='sz=100x100')
self.ae(r.status, http_client.OK)
self.ae(r.getheader('Used-Cache'), 'yes')
change_cover(1, 1)
r, data = get('thumb', 1, q='sz=100')
self.ae(r.status, http_client.OK)
self.ae(identify(data), ('jpeg', 100, 100))
self.ae(r.getheader('Used-Cache'), 'no')
# Test file sharing in cache
r, data = get('cover', 2)
self.ae(r.status, http_client.OK)
self.ae(data, db.cover(2))
self.ae(r.getheader('Used-Cache'), 'no')
path = from_hex_unicode(r.getheader('Tempfile'))
f, fdata = share_open(path, 'rb'), data
# Now force an update
change_cover(1)
r, data = get('cover', 2)
self.ae(r.status, http_client.OK)
self.ae(data, db.cover(2))
self.ae(r.getheader('Used-Cache'), 'no')
path = from_hex_unicode(r.getheader('Tempfile'))
f2, f2data = share_open(path, 'rb'), data
# Do it again
change_cover(2)
r, data = get('cover', 2)
self.ae(r.status, http_client.OK)
self.ae(data, db.cover(2))
self.ae(r.getheader('Used-Cache'), 'no')
self.ae(f.read(), fdata)
self.ae(f2.read(), f2data)
# Test serving of metadata as opf
r, data = get('opf', 1)
self.ae(r.status, http_client.OK)
self.ae(r.getheader('Content-Type'),
'application/oebps-package+xml; charset=UTF-8')
self.assertIsNotNone(r.getheader('Last-Modified'))
opf = OPF(BytesIO(data),
populate_spine=False,
try_to_guess_cover=False)
self.ae(db.field_for('title', 1), opf.title)
self.ae(db.field_for('authors', 1), tuple(opf.authors))
conn.request('GET',
'/get/opf/1',
headers={'Accept-Encoding': 'gzip'})
r = conn.getresponse()
self.ae(r.status,
http_client.OK), self.ae(r.getheader('Content-Encoding'),
'gzip')
raw = r.read()
self.ae(zlib.decompress(raw, 16 + zlib.MAX_WBITS), data)
# Test serving metadata as json
r, data = get('json', 1)
self.ae(r.status, http_client.OK)
self.ae(db.field_for('title', 1), json.loads(data)['title'])
conn.request('GET',
'/get/json/1',
headers={'Accept-Encoding': 'gzip'})
r = conn.getresponse()
self.ae(r.status,
http_client.OK), self.ae(r.getheader('Content-Encoding'),
'gzip')
raw = r.read()
self.ae(zlib.decompress(raw, 16 + zlib.MAX_WBITS), data)
def _generate_thumbnail(self, book):
'''
Fetch the cover image, generate a thumbnail, cache
Extracts covers from zipped epubs
'''
self._log_location(book.title)
#self._log("book_path: %s" % book.path)
#self._log("book: '%s' by %s uuid: %s" % (book.title, book.author, book.uuid))
# Parse the cover from the connected device, model Fetch_Annotations:_get_epub_toc()
thumb_data = None
thumb_path = book.path.rpartition('.')[0] + '.jpg'
# Try getting the cover from the cache
try:
zfr = ZipFile(self.archive_path)
thumb_data = zfr.read(thumb_path)
if thumb_data == 'None':
self._log("returning None from cover cache")
zfr.close()
return None
except:
self._log("opening cover cache for appending")
zfw = ZipFile(self.archive_path, mode='a')
else:
self._log("returning thumb from cover cache")
return thumb_data
# Get the cover from the book
try:
stream = cStringIO.StringIO(self.ios.read(book.path, mode='rb'))
mi = get_metadata(stream)
if mi.cover_data is not None:
img_data = cStringIO.StringIO(mi.cover_data[1])
except:
if self.verbose:
self._log("ERROR: unable to get cover from '%s'" % book.title)
import traceback
#traceback.print_exc()
exc_type, exc_value, exc_traceback = sys.exc_info()
self._log(traceback.format_exception_only(exc_type, exc_value)[0].strip())
return thumb_data
# Generate a thumb
try:
im = PILImage.open(img_data)
scaled, width, height = fit_image(im.size[0], im.size[1], 60, 80)
im = im.resize((int(width), int(height)), PILImage.ANTIALIAS)
thumb = cStringIO.StringIO()
im.convert('RGB').save(thumb, 'JPEG')
thumb_data = thumb.getvalue()
thumb.close()
self._log("SUCCESS: generated thumb for '%s', caching" % book.title)
# Cache the tagged thumb
zfw.writestr(thumb_path, thumb_data)
except:
if self.verbose:
self._log("ERROR generating thumb for '%s', caching empty marker" % book.title)
import traceback
exc_type, exc_value, exc_traceback = sys.exc_info()
self._log(traceback.format_exception_only(exc_type, exc_value)[0].strip())
# Cache the empty cover
zfw.writestr(thumb_path, 'None')
finally:
img_data.close()
zfw.close()
return thumb_data
def test_get(self): # {{{
'Test /get'
with self.create_server() as server:
db = server.handler.router.ctx.get_library()
conn = server.connect()
def get(what, book_id, library_id=None, q=''):
q = ('?' + q) if q else q
conn.request('GET', '/get/%s/%s' % (what, book_id) + (('/' + library_id) if library_id else '') + q)
r = conn.getresponse()
return r, r.read()
# Test various invalid parameters
def bad(*args):
r, data = get(*args)
self.ae(r.status, httplib.NOT_FOUND)
bad('xxx', 1)
bad('fmt1', 10)
bad('fmt1', 1, 'zzzz')
bad('fmt1', 'xx')
# Test simple fetching of format without metadata update
r, data = get('fmt1', 1, db.server_library_id)
self.ae(data, db.format(1, 'fmt1'))
self.assertIsNotNone(r.getheader('Content-Disposition'))
self.ae(r.getheader('Used-Cache'), 'no')
r, data = get('fmt1', 1)
self.ae(data, db.format(1, 'fmt1'))
self.ae(r.getheader('Used-Cache'), 'yes')
# Test fetching of format with metadata update
raw = P('quick_start/eng.epub', data=True)
r, data = get('epub', 1)
self.ae(r.status, httplib.OK)
etag = r.getheader('ETag')
self.assertIsNotNone(etag)
self.ae(r.getheader('Used-Cache'), 'no')
self.assertTrue(data.startswith(b'PK'))
self.assertGreaterEqual(len(data), len(raw))
db.set_field('title', {1:'changed'})
r, data = get('epub', 1)
self.assertNotEqual(r.getheader('ETag'), etag)
etag = r.getheader('ETag')
self.ae(r.getheader('Used-Cache'), 'no')
mi = get_metadata(BytesIO(data), extract_cover=False)
self.ae(mi.title, 'changed')
r, data = get('epub', 1)
self.ae(r.getheader('Used-Cache'), 'yes')
# Test plugboards
import calibre.library.save_to_disk as c
orig, c.DEBUG = c.DEBUG, False
try:
db.set_pref('plugboards', {u'epub': {u'content_server': [[u'changed, {title}', u'title']]}})
# this is needed as the cache is not invalidated for plugboard changes
db.set_field('title', {1:'again'})
r, data = get('epub', 1)
self.assertNotEqual(r.getheader('ETag'), etag)
etag = r.getheader('ETag')
self.ae(r.getheader('Used-Cache'), 'no')
mi = get_metadata(BytesIO(data), extract_cover=False)
self.ae(mi.title, 'changed, again')
finally:
c.DEBUG = orig
# Test the serving of covers
r, data = get('cover', 1)
self.ae(r.status, httplib.OK)
self.ae(data, db.cover(1))
self.ae(r.getheader('Used-Cache'), 'no')
self.ae(r.getheader('Content-Type'), 'image/jpeg')
r, data = get('cover', 1)
self.ae(r.status, httplib.OK)
self.ae(data, db.cover(1))
self.ae(r.getheader('Used-Cache'), 'yes')
r, data = get('cover', 3)
self.ae(r.status, httplib.NOT_FOUND)
r, data = get('thumb', 1)
self.ae(r.status, httplib.OK)
self.ae(identify_data(data), (60, 60, 'jpeg'))
self.ae(r.getheader('Used-Cache'), 'no')
r, data = get('thumb', 1)
self.ae(r.status, httplib.OK)
self.ae(r.getheader('Used-Cache'), 'yes')
r, data = get('thumb', 1, q='sz=100')
self.ae(r.status, httplib.OK)
self.ae(identify_data(data), (100, 100, 'jpeg'))
self.ae(r.getheader('Used-Cache'), 'no')
r, data = get('thumb', 1, q='sz=100x100')
self.ae(r.status, httplib.OK)
self.ae(r.getheader('Used-Cache'), 'yes')
db.set_cover({1:I('lt.png', data=True)})
r, data = get('thumb', 1, q='sz=100')
self.ae(r.status, httplib.OK)
self.ae(identify_data(data), (100, 100, 'jpeg'))
self.ae(r.getheader('Used-Cache'), 'no')
# Test serving of metadata as opf
r, data = get('opf', 1)
self.ae(r.status, httplib.OK)
self.ae(r.getheader('Content-Type'), 'application/oebps-package+xml; charset=UTF-8')
self.assertIsNotNone(r.getheader('Last-Modified'))
opf = OPF(BytesIO(data), populate_spine=False, try_to_guess_cover=False)
self.ae(db.field_for('title', 1), opf.title)
self.ae(db.field_for('authors', 1), tuple(opf.authors))
conn.request('GET', '/get/opf/1', headers={'Accept-Encoding':'gzip'})
r = conn.getresponse()
self.ae(r.status, httplib.OK), self.ae(r.getheader('Content-Encoding'), 'gzip')
raw = r.read()
self.ae(zlib.decompress(raw, 16+zlib.MAX_WBITS), data)
# Test serving metadata as json
r, data = get('json', 1)
self.ae(r.status, httplib.OK)
self.ae(db.field_for('title', 1), json.loads(data)['title'])
conn.request('GET', '/get/json/1', headers={'Accept-Encoding':'gzip'})
r = conn.getresponse()
self.ae(r.status, httplib.OK), self.ae(r.getheader('Content-Encoding'), 'gzip')
raw = r.read()
self.ae(zlib.decompress(raw, 16+zlib.MAX_WBITS), data)
