def collect_data(self, ids, tdir):
from calibre.ebooks.metadata.opf2 import metadata_to_opf
data = {}
for i in set(ids):
mi = self.db.get_metadata(i,
index_is_id=True,
get_cover=True,
cover_as_data=True)
opf = metadata_to_opf(mi)
if isbytestring(opf):
opf = opf.decode('utf-8')
cpath = None
if mi.cover_data and mi.cover_data[1]:
cpath = os.path.join(tdir, 'cover_%s.jpg' % i)
with lopen(cpath, 'wb') as f:
f.write(mi.cover_data[1])
if isbytestring(cpath):
cpath = cpath.decode(filesystem_encoding)
formats = {}
if mi.formats:
for fmt in mi.formats:
fpath = os.path.join(tdir, 'fmt_%s.%s' % (i, fmt.lower()))
with lopen(fpath, 'wb') as f:
try:
self.db.copy_format_to(i, fmt, f, index_is_id=True)
except NoSuchFormat:
continue
else:
if isbytestring(fpath):
fpath = fpath.decode(filesystem_encoding)
formats[fmt.lower()] = fpath
data[i] = [opf, cpath, formats, mi.last_modified.isoformat()]
return data
def collect_data(self, ids, tdir):
from calibre.ebooks.metadata.opf2 import metadata_to_opf
data = {}
for i in set(ids):
mi = self.db.get_metadata(i, index_is_id=True, get_cover=True,
cover_as_data=True)
opf = metadata_to_opf(mi)
if isbytestring(opf):
opf = opf.decode('utf-8')
cpath = None
if mi.cover_data and mi.cover_data[1]:
cpath = os.path.join(tdir, 'cover_%s.jpg'%i)
with lopen(cpath, 'wb') as f:
f.write(mi.cover_data[1])
if isbytestring(cpath):
cpath = cpath.decode(filesystem_encoding)
formats = {}
if mi.formats:
for fmt in mi.formats:
fpath = os.path.join(tdir, 'fmt_%s.%s'%(i, fmt.lower()))
with lopen(fpath, 'wb') as f:
try:
self.db.copy_format_to(i, fmt, f, index_is_id=True)
except NoSuchFormat:
continue
else:
if isbytestring(fpath):
fpath = fpath.decode(filesystem_encoding)
formats[fmt.lower()] = fpath
data[i] = [opf, cpath, formats, mi.last_modified.isoformat()]
return data
def get_metadata_as_opf(self, id_):
self.set_header('Content-Type',
'application/oebps-package+xml; charset=UTF-8')
mi = self.db.get_metadata(id_, index_is_id=True)
data = metadata_to_opf(mi)
self.set_header('Last-Modified', self.last_modified(mi.last_modified))
return data
def dump_metadata(self, book_ids=None, remove_from_dirtied=True,
callback=None):
'''
Write metadata for each record to an individual OPF file. If callback
is not None, it is called once at the start with the number of book_ids
being processed. And once for every book_id, with arguments (book_id,
mi, ok).
'''
if book_ids is None:
book_ids = set(self.dirtied_cache)
if callback is not None:
callback(len(book_ids), True, False)
for book_id in book_ids:
if self._field_for('path', book_id) is None:
if callback is not None:
callback(book_id, None, False)
continue
mi, sequence = self._get_metadata_for_dump(book_id)
if mi is None:
if callback is not None:
callback(book_id, mi, False)
continue
try:
raw = metadata_to_opf(mi)
self._write_backup(book_id, raw)
if remove_from_dirtied:
self._clear_dirtied(book_id, sequence)
except:
pass
if callback is not None:
callback(book_id, mi, True)
def single_identify(title, authors, identifiers):
log = GUILog()
patch_plugins()
results = identify(log, Event(), title=title, authors=authors,
identifiers=identifiers)
return [metadata_to_opf(r) for r in results], [r.has_cached_cover_url for
r in results], dump_caches(), log.dump()
def add(self, book_id, mi, formats, one_liner=''):
self.one_liner = one_liner
self.title = mi.title
# authors
authors = []
for x in mi.authors:
authors.append(author_to_author_sort(x))
self.authors = '|'.join(authors)
# issues
issues = []
um = mi.get_all_user_metadata(False)
if '#issue' in um:
issue_strs = um['#issue']['#value#']
for issue_str in issue_strs:
issue_id = issue_str.rpartition('(')[-1].partition(')')[0]
issues.append(issue_id)
self.issues = '|'.join(issues)
# etc
self.description = mi.comments
# opf
self.opf = metadata_to_opf(mi)
# file to upload
for format, file_loc in formats.items():
self.file = file_loc
# metadata
self.mi = mi
self.book_id = book_id
self._start()
def queue_job(ctx, rd, library_id, db, fmt, book_id, conversion_data):
from calibre.ebooks.metadata.opf2 import metadata_to_opf
from calibre.ebooks.conversion.config import GuiRecommendations, save_specifics
from calibre.customize.conversion import OptionRecommendation
tdir = tempfile.mkdtemp(dir=rd.tdir)
with tempfile.NamedTemporaryFile(prefix='', suffix=('.' + fmt.lower()), dir=tdir, delete=False) as src_file:
db.copy_format_to(book_id, fmt, src_file)
with tempfile.NamedTemporaryFile(prefix='', suffix='.jpg', dir=tdir, delete=False) as cover_file:
cover_copied = db.copy_cover_to(book_id, cover_file)
cover_path = cover_file.name if cover_copied else None
mi = db.get_metadata(book_id)
mi.application_id = mi.uuid
raw = metadata_to_opf(mi)
with tempfile.NamedTemporaryFile(prefix='', suffix='.opf', dir=tdir, delete=False) as opf_file:
opf_file.write(raw)
recs = GuiRecommendations()
recs.update(conversion_data['options'])
recs['gui_preferred_input_format'] = conversion_data['input_fmt'].lower()
save_specifics(db, book_id, recs)
recs = [(k, v, OptionRecommendation.HIGH) for k, v in recs.iteritems()]
job_id = ctx.start_job(
'Convert book %s (%s)' % (book_id, fmt), 'calibre.srv.convert',
'convert_book', args=(
src_file.name, opf_file.name, cover_path, conversion_data['output_fmt'], recs),
job_done_callback=job_done
)
expire_old_jobs()
with cache_lock:
conversion_jobs[job_id] = JobStatus(
job_id, book_id, tdir, library_id, src_file.name, conversion_data)
return job_id
def create_book(mi, path, fmt='epub', opf_name='metadata.opf', html_name='start.xhtml', toc_name='toc.ncx'):
''' Create an empty book in the specified format at the specified location. '''
path = os.path.abspath(path)
lang = 'und'
opf = metadata_to_opf(mi, as_string=False)
for l in opf.xpath('//*[local-name()="language"]'):
if l.text:
lang = l.text
break
lang = lang_as_iso639_1(lang) or lang
opfns = OPF_NAMESPACES['opf']
m = opf.makeelement('{%s}manifest' % opfns)
opf.insert(1, m)
i = m.makeelement('{%s}item' % opfns, href=html_name, id='start')
i.set('media-type', guess_type('a.xhtml'))
m.append(i)
i = m.makeelement('{%s}item' % opfns, href=toc_name, id='ncx')
i.set('media-type', guess_type(toc_name))
m.append(i)
s = opf.makeelement('{%s}spine' % opfns, toc="ncx")
opf.insert(2, s)
i = s.makeelement('{%s}itemref' % opfns, idref='start')
s.append(i)
CONTAINER = '''\
<?xml version="1.0"?>
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
<rootfiles>
<rootfile full-path="{0}" media-type="application/oebps-package+xml"/>
</rootfiles>
</container>
'''.format(prepare_string_for_xml(opf_name, True)).encode('utf-8')
HTML = P('templates/new_book.html', data=True).decode('utf-8').replace(
'_LANGUAGE_', prepare_string_for_xml(lang, True)
).replace(
'_TITLE_', prepare_string_for_xml(mi.title)
).replace(
'_AUTHORS_', prepare_string_for_xml(authors_to_string(mi.authors))
).encode('utf-8')
h = parse(HTML)
pretty_html_tree(None, h)
HTML = serialize(h, 'text/html')
ncx = etree.tostring(create_toc(mi, opf, html_name, lang), encoding='utf-8', xml_declaration=True, pretty_print=True)
pretty_xml_tree(opf)
opf = etree.tostring(opf, encoding='utf-8', xml_declaration=True, pretty_print=True)
if fmt == 'azw3':
with TemporaryDirectory('create-azw3') as tdir, CurrentDir(tdir):
for name, data in ((opf_name, opf), (html_name, HTML), (toc_name, ncx)):
with open(name, 'wb') as f:
f.write(data)
c = Container(os.path.dirname(os.path.abspath(opf_name)), opf_name, DevNull())
opf_to_azw3(opf_name, path, c)
else:
with ZipFile(path, 'w', compression=ZIP_STORED) as zf:
zf.writestr('mimetype', b'application/epub+zip', compression=ZIP_STORED)
zf.writestr('META-INF/', b'', 0755)
zf.writestr('META-INF/container.xml', CONTAINER)
zf.writestr(opf_name, opf)
zf.writestr(html_name, HTML)
zf.writestr(toc_name, ncx)
def do_one(self):
try:
book_id = self.db.get_a_dirtied_book()
if book_id is None:
return
except Abort:
raise
except:
# Happens during interpreter shutdown
return
self.wait(0)
try:
mi, sequence = self.db.get_metadata_for_dump(book_id)
except:
prints('Failed to get backup metadata for id:', book_id, 'once')
traceback.print_exc()
self.wait(self.interval)
try:
mi, sequence = self.db.get_metadata_for_dump(book_id)
except:
prints('Failed to get backup metadata for id:', book_id,
'again, giving up')
traceback.print_exc()
return
if mi is None:
self.db.clear_dirtied(book_id, sequence)
return
# Give the GUI thread a chance to do something. Python threads don't
# have priorities, so this thread would naturally keep the processor
# until some scheduling event happens. The wait makes such an event
self.wait(self.scheduling_interval)
try:
raw = metadata_to_opf(mi)
except:
prints('Failed to convert to opf for id:', book_id)
traceback.print_exc()
return
self.wait(self.scheduling_interval)
try:
self.db.write_backup(book_id, raw)
except:
prints('Failed to write backup metadata for id:', book_id, 'once')
traceback.print_exc()
self.wait(self.interval)
try:
self.db.write_backup(book_id, raw)
except:
prints('Failed to write backup metadata for id:', book_id,
'again, giving up')
traceback.print_exc()
return
self.db.clear_dirtied(book_id, sequence)
def single_identify(title, authors, identifiers):
log = GUILog()
patch_plugins()
results = identify(log, Event(), title=title, authors=authors,
identifiers=identifiers)
return [metadata_to_opf(r) for r in results], [r.has_cached_cover_url for
r in results], dump_caches(), log.dump()
def copy_metadata(self):
rows = self.gui.library_view.selectionModel().selectedRows()
if not rows or len(rows) == 0:
return error_dialog(self.gui,
_('Cannot copy metadata'),
_('No books selected'),
show=True)
if len(rows) > 1:
return error_dialog(
self.gui,
_('Cannot copy metadata'),
_('Multiple books selected, can only copy from one book at a time.'
),
show=True)
db = self.gui.current_db
book_id = db.id(rows[0].row())
mi = db.new_api.get_metadata(book_id)
md = QMimeData()
md.setText(str(mi))
md.setData('application/calibre-book-metadata',
bytearray(metadata_to_opf(mi, default_lang='und')))
img = db.new_api.cover(book_id, as_image=True)
if img:
md.setImageData(img)
c = QApplication.clipboard()
c.setMimeData(md)
def add(self, book_id, mi, formats, one_liner=''):
self.one_liner = one_liner
self.title = mi.title
# authors
authors = []
for x in mi.authors:
authors.append(author_to_author_sort(x))
self.authors = '|'.join(authors)
# issues
issues = []
um = mi.get_all_user_metadata(False)
if '#issue' in um:
issue_strs = um['#issue']['#value#']
for issue_str in issue_strs:
issue_id = issue_str.rpartition('(')[-1].partition(')')[0]
issues.append(issue_id)
self.issues = '|'.join(issues)
# etc
self.description = mi.comments
# opf
self.opf = metadata_to_opf(mi)
# file to upload
for format, file_loc in formats.items():
self.file = file_loc
# metadata
self.mi = mi
self.book_id = book_id
self._start()
def dump_metadata(self,
book_ids=None,
remove_from_dirtied=True,
callback=None):
'''
Write metadata for each record to an individual OPF file. If callback
is not None, it is called once at the start with the number of book_ids
being processed. And once for every book_id, with arguments (book_id,
mi, ok).
'''
if book_ids is None:
book_ids = set(self.dirtied_cache)
if callback is not None:
callback(len(book_ids), True, False)
for book_id in book_ids:
if self._field_for('path', book_id) is None:
if callback is not None:
callback(book_id, None, False)
continue
mi, sequence = self._get_metadata_for_dump(book_id)
if mi is None:
if callback is not None:
callback(book_id, mi, False)
continue
try:
raw = metadata_to_opf(mi)
self._write_backup(book_id, raw)
if remove_from_dirtied:
self._clear_dirtied(book_id, sequence)
except:
pass
if callback is not None:
callback(book_id, mi, True)
def do_one(self):
try:
book_id = self.db.get_a_dirtied_book()
if book_id is None:
return
except Abort:
raise
except:
# Happens during interpreter shutdown
return
self.wait(0)
try:
mi, sequence = self.db.get_metadata_for_dump(book_id)
except:
prints('Failed to get backup metadata for id:', book_id, 'once')
traceback.print_exc()
self.wait(self.interval)
try:
mi, sequence = self.db.get_metadata_for_dump(book_id)
except:
prints('Failed to get backup metadata for id:', book_id, 'again, giving up')
traceback.print_exc()
return
if mi is None:
self.db.clear_dirtied(book_id, sequence)
return
# Give the GUI thread a chance to do something. Python threads don't
# have priorities, so this thread would naturally keep the processor
# until some scheduling event happens. The wait makes such an event
self.wait(self.scheduling_interval)
try:
raw = metadata_to_opf(mi)
except:
prints('Failed to convert to opf for id:', book_id)
traceback.print_exc()
self.db.clear_dirtied(book_id, sequence)
return
self.wait(self.scheduling_interval)
try:
self.db.write_backup(book_id, raw)
except:
prints('Failed to write backup metadata for id:', book_id, 'once')
traceback.print_exc()
self.wait(self.interval)
try:
self.db.write_backup(book_id, raw)
except:
prints('Failed to write backup metadata for id:', book_id, 'again, giving up')
traceback.print_exc()
return
self.db.clear_dirtied(book_id, sequence)
def main(args=sys.argv):
parser = option_parser()
opts, args = parser.parse_args(args)
buf = BytesIO()
log = create_log(buf)
abort = Event()
patch_plugins()
authors = []
if opts.authors:
authors = string_to_authors(opts.authors)
identifiers = {}
if opts.isbn:
identifiers['isbn'] = opts.isbn
allowed_plugins = frozenset(opts.allowed_plugin)
results = identify(log,
abort,
title=opts.title,
authors=authors,
identifiers=identifiers,
timeout=int(opts.timeout),
allowed_plugins=allowed_plugins or None)
if not results:
print(log, file=sys.stderr)
prints('No results found', file=sys.stderr)
raise SystemExit(1)
result = results[0]
cf = None
if opts.cover and results:
cover = download_cover(log,
title=opts.title,
authors=authors,
identifiers=result.identifiers,
timeout=int(opts.timeout))
if cover is None and not opts.opf:
prints('No cover found', file=sys.stderr)
else:
save_cover_data_to(cover[-1], opts.cover)
result.cover = cf = opts.cover
log = buf.getvalue()
result = (metadata_to_opf(result)
if opts.opf else unicode(result).encode('utf-8'))
if opts.verbose:
print(log, file=sys.stderr)
print(result)
if not opts.opf and opts.cover:
prints('Cover :', cf)
return 0
def main(args=sys.argv):
parser = option_parser()
opts, args = parser.parse_args(args)
buf = BytesIO()
log = create_log(buf)
abort = Event()
patch_plugins()
authors = []
if opts.authors:
authors = string_to_authors(opts.authors)
identifiers = {}
for idspec in opts.identifier:
k, v = idspec.partition(':')[::2]
if not k or not v:
raise SystemExit('Not a valid identifier: {}'.format(idspec))
identifiers[k] = v
if opts.isbn:
identifiers['isbn'] = opts.isbn
allowed_plugins = frozenset(opts.allowed_plugin)
results = identify(log, abort, title=opts.title, authors=authors,
identifiers=identifiers, timeout=int(opts.timeout),
allowed_plugins=allowed_plugins or None)
if not results:
print(log, file=sys.stderr)
prints('No results found', file=sys.stderr)
raise SystemExit(1)
result = results[0]
cf = None
if opts.cover and results:
cover = download_cover(log, title=opts.title, authors=authors,
identifiers=result.identifiers, timeout=int(opts.timeout))
if cover is None and not opts.opf:
prints('No cover found', file=sys.stderr)
else:
save_cover_data_to(cover[-1], opts.cover)
result.cover = cf = opts.cover
log = buf.getvalue()
result = (metadata_to_opf(result) if opts.opf else
unicode_type(result).encode('utf-8'))
if opts.verbose:
print(log, file=sys.stderr)
print(result)
if not opts.opf and opts.cover:
prints('Cover :', cf)
return 0
def main(args=sys.argv):
parser = option_parser()
opts, args = parser.parse_args(args)
buf = BytesIO()
log = create_log(buf)
abort = Event()
patch_plugins()
authors = []
if opts.authors:
authors = string_to_authors(opts.authors)
identifiers = {}
for idspec in opts.identifier:
k, v = idspec.partition(':')[::2]
if not k or not v:
raise SystemExit('Not a valid identifier: {}'.format(idspec))
identifiers[k] = v
if opts.isbn:
identifiers['isbn'] = opts.isbn
allowed_plugins = frozenset(opts.allowed_plugin)
results = identify(log, abort, title=opts.title, authors=authors,
identifiers=identifiers, timeout=int(opts.timeout),
allowed_plugins=allowed_plugins or None)
if not results:
prints(buf.getvalue(), file=sys.stderr)
prints('No results found', file=sys.stderr)
raise SystemExit(1)
result = results[0]
cf = None
if opts.cover and results:
cover = download_cover(log, title=opts.title, authors=authors,
identifiers=result.identifiers, timeout=int(opts.timeout))
if cover is None:
if not opts.opf:
prints('No cover found', file=sys.stderr)
else:
save_cover_data_to(cover[-1], opts.cover)
result.cover = cf = opts.cover
if opts.verbose:
prints(buf.getvalue(), file=sys.stderr)
if opts.opf:
getattr(sys.stdout, 'buffer', sys.stdout).write(metadata_to_opf(result))
print()
else:
prints(str(result))
if not opts.opf and opts.cover:
prints('Cover :', cf)
return 0
def get(ctx, rd, what, book_id, library_id):
book_id, rest = book_id.partition('_')[::2]
try:
book_id = int(book_id)
except Exception:
raise HTTPNotFound('Book with id %r does not exist' % book_id)
db = get_db(ctx, rd, library_id)
if db is None:
raise HTTPNotFound('Library %r not found' % library_id)
with db.safe_read_lock:
if not ctx.has_id(rd, db, book_id):
raise BookNotFound(book_id, db)
library_id = db.server_library_id # in case library_id was None
if what == 'thumb':
sz = rd.query.get('sz')
w, h = 60, 80
if sz is None:
if rest:
try:
w, h = map(int, rest.split('_'))
except Exception:
pass
elif sz == 'full':
w = h = None
elif 'x' in sz:
try:
w, h = map(int, sz.partition('x')[::2])
except Exception:
pass
else:
try:
w = h = int(sz)
except Exception:
pass
return cover(ctx, rd, library_id, db, book_id, width=w, height=h)
elif what == 'cover':
return cover(ctx, rd, library_id, db, book_id)
elif what == 'opf':
mi = db.get_metadata(book_id, get_cover=False)
rd.outheaders[
'Content-Type'] = 'application/oebps-package+xml; charset=UTF-8'
rd.outheaders['Last-Modified'] = http_date(
timestampfromdt(mi.last_modified))
return metadata_to_opf(mi)
elif what == 'json':
from calibre.srv.ajax import book_to_json
data, last_modified = book_to_json(ctx, rd, db, book_id)
rd.outheaders['Last-Modified'] = http_date(
timestampfromdt(last_modified))
return json(ctx, rd, get, data)
else:
try:
return book_fmt(ctx, rd, library_id, db, book_id, what.lower())
except NoSuchFormat:
raise HTTPNotFound('No %s format for the book %r' %
(what.lower(), book_id))
def get_metadata_as_opf(self, id_):
cherrypy.response.headers['Content-Type'] = \
'application/oebps-package+xml; charset=UTF-8'
mi = self.db.get_metadata(id_, index_is_id=True)
data = metadata_to_opf(mi)
cherrypy.response.timeout = 3600
cherrypy.response.headers['Last-Modified'] = \
self.last_modified(mi.last_modified)
return data
def get_metadata_as_opf(self, id_):
cherrypy.response.headers['Content-Type'] = \
'application/oebps-package+xml; charset=UTF-8'
mi = self.db.get_metadata(id_, index_is_id=True)
data = metadata_to_opf(mi)
cherrypy.response.timeout = 3600
cherrypy.response.headers['Last-Modified'] = \
self.last_modified(mi.last_modified)
return data
def create_opf_file(db, book_id):
mi = db.get_metadata(book_id, index_is_id=True)
mi.application_id = uuid.uuid4()
old_cover = mi.cover
mi.cover = None
raw = metadata_to_opf(mi)
mi.cover = old_cover
opf_file = PersistentTemporaryFile('.opf')
opf_file.write(raw)
opf_file.close()
return mi, opf_file
def create_opf_file(db, book_id):
mi = db.get_metadata(book_id, index_is_id=True)
mi.application_id = uuid.uuid4()
old_cover = mi.cover
mi.cover = None
raw = metadata_to_opf(mi)
mi.cover = old_cover
opf_file = PersistentTemporaryFile('.opf')
opf_file.write(raw)
opf_file.close()
return mi, opf_file
def main(do_identify, covers, metadata, ensure_fields, tdir):
failed_ids = set()
failed_covers = set()
all_failed = True
log = GUILog()
patch_plugins()
for book_id, mi in iteritems(metadata):
mi = OPF(BytesIO(mi), basedir=tdir,
populate_spine=False).to_book_metadata()
title, authors, identifiers = mi.title, mi.authors, mi.identifiers
cdata = None
log.clear()
if do_identify:
results = []
try:
results = identify(log,
Event(),
title=title,
authors=authors,
identifiers=identifiers)
except:
pass
if results:
all_failed = False
mi = merge_result(mi, results[0], ensure_fields=ensure_fields)
identifiers = mi.identifiers
if not mi.is_null('rating'):
# set_metadata expects a rating out of 10
mi.rating *= 2
with open(os.path.join(tdir, '%d.mi' % book_id), 'wb') as f:
f.write(metadata_to_opf(mi, default_lang='und'))
else:
log.error('Failed to download metadata for', title)
failed_ids.add(book_id)
if covers:
cdata = download_cover(log,
title=title,
authors=authors,
identifiers=identifiers)
if cdata is None:
failed_covers.add(book_id)
else:
with open(os.path.join(tdir, '%d.cover' % book_id), 'wb') as f:
f.write(cdata[-1])
all_failed = False
with open(os.path.join(tdir, '%d.log' % book_id), 'wb') as f:
f.write(log.plain_text.encode('utf-8'))
return failed_ids, failed_covers, all_failed
def _print_result(self, result, ranking, opf=False):
if opf:
result_text = metadata_to_opf(result)
else:
if result.pubdate:
pubdate = str(result.pubdate.date())
else:
pubdate = 'Unknown'
result_text = '(%04d) - %s: %s [%s]' % (
ranking(result), result.identifiers['comicvine'],
result.title, pubdate)
print result_text
def commit(self, book_id):
''' Commits any local changes to the metadata for this book up to the Casanova server '''
# get the opf that will be posted
mi = self.db.get_metadata(book_id, index_is_id=True, get_cover=True)
opf = metadata_to_opf(mi)
# get the remote id
try:
casanova_id = mi.identifiers['casanova']
except AttributeError:
print('There is no Casanova identifier for this book')
# now post the metadata
return self._post_metadata(casanova_id, opf, mi.cover)
def commit(self, book_id):
''' Commits any local changes to the metadata for this book up to the Casanova server '''
# get the opf that will be posted
mi = self.db.get_metadata(book_id, index_is_id=True, get_cover=True)
opf = metadata_to_opf(mi)
# get the remote id
try:
casanova_id = mi.identifiers['casanova']
except AttributeError:
print('There is no Casanova identifier for this book')
# now post the metadata
return self._post_metadata(casanova_id, opf, mi.cover)
def get(ctx, rd, what, book_id, library_id):
book_id, rest = book_id.partition('_')[::2]
try:
book_id = int(book_id)
except Exception:
raise HTTPNotFound('Book with id %r does not exist' % book_id)
db = get_db(ctx, rd, library_id)
if db is None:
raise HTTPNotFound('Library %r not found' % library_id)
with db.safe_read_lock:
if not ctx.has_id(rd, db, book_id):
raise BookNotFound(book_id, db)
library_id = db.server_library_id # in case library_id was None
if what == 'thumb':
sz = rd.query.get('sz')
w, h = 60, 80
if sz is None:
if rest:
try:
w, h = map(int, rest.split('_'))
except Exception:
pass
elif sz == 'full':
w = h = None
elif 'x' in sz:
try:
w, h = map(int, sz.partition('x')[::2])
except Exception:
pass
else:
try:
w = h = int(sz)
except Exception:
pass
return cover(ctx, rd, library_id, db, book_id, width=w, height=h)
elif what == 'cover':
return cover(ctx, rd, library_id, db, book_id)
elif what == 'opf':
mi = db.get_metadata(book_id, get_cover=False)
rd.outheaders['Content-Type'] = 'application/oebps-package+xml; charset=UTF-8'
rd.outheaders['Last-Modified'] = http_date(timestampfromdt(mi.last_modified))
return metadata_to_opf(mi)
elif what == 'json':
from calibre.srv.ajax import book_to_json
data, last_modified = book_to_json(ctx, rd, db, book_id)
rd.outheaders['Last-Modified'] = http_date(timestampfromdt(last_modified))
return json(ctx, rd, get, data)
else:
try:
return book_fmt(ctx, rd, library_id, db, book_id, what.lower())
except NoSuchFormat:
raise HTTPNotFound('No %s format for the book %r' % (what.lower(), book_id))
def do_save_book_to_disk(db, book_id, mi, plugboards,
formats, root, opts, length):
originals = mi.cover, mi.pubdate, mi.timestamp
formats_written = False
try:
if mi.pubdate:
mi.pubdate = as_local_time(mi.pubdate)
if mi.timestamp:
mi.timestamp = as_local_time(mi.timestamp)
components = get_path_components(opts, mi, book_id, length)
base_path = os.path.join(root, *components)
base_name = os.path.basename(base_path)
dirpath = os.path.dirname(base_path)
try:
os.makedirs(dirpath)
except EnvironmentError as err:
if err.errno != errno.EEXIST:
raise
cdata = None
if opts.save_cover:
cdata = db.cover(book_id)
if cdata:
cpath = base_path + '.jpg'
with lopen(cpath, 'wb') as f:
f.write(cdata)
mi.cover = base_name+'.jpg'
if opts.write_opf:
from calibre.ebooks.metadata.opf2 import metadata_to_opf
opf = metadata_to_opf(mi)
with lopen(base_path+'.opf', 'wb') as f:
f.write(opf)
finally:
mi.cover, mi.pubdate, mi.timestamp = originals
if not formats:
return not formats_written, book_id, mi.title
for fmt in formats:
fmt_path = base_path+'.'+unicode_type(fmt)
try:
db.copy_format_to(book_id, fmt, fmt_path)
formats_written = True
except NoSuchFormat:
continue
if opts.update_metadata:
with lopen(fmt_path, 'r+b') as stream:
update_metadata(mi, fmt, stream, plugboards, cdata)
return not formats_written, book_id, mi.title
def do_save_book_to_disk(db, book_id, mi, plugboards,
formats, root, opts, length):
originals = mi.cover, mi.pubdate, mi.timestamp
formats_written = False
try:
if mi.pubdate:
mi.pubdate = as_local_time(mi.pubdate)
if mi.timestamp:
mi.timestamp = as_local_time(mi.timestamp)
components = get_path_components(opts, mi, book_id, length)
base_path = os.path.join(root, *components)
base_name = os.path.basename(base_path)
dirpath = os.path.dirname(base_path)
try:
os.makedirs(dirpath)
except EnvironmentError as err:
if err.errno != errno.EEXIST:
raise
cdata = None
if opts.save_cover:
cdata = db.cover(book_id)
if cdata:
cpath = base_path + '.jpg'
with lopen(cpath, 'wb') as f:
f.write(cdata)
mi.cover = base_name+'.jpg'
if opts.write_opf:
from calibre.ebooks.metadata.opf2 import metadata_to_opf
opf = metadata_to_opf(mi)
with lopen(base_path+'.opf', 'wb') as f:
f.write(opf)
finally:
mi.cover, mi.pubdate, mi.timestamp = originals
if not formats:
return not formats_written, book_id, mi.title
for fmt in formats:
fmt_path = base_path+'.'+str(fmt)
try:
db.copy_format_to(book_id, fmt, fmt_path)
formats_written = True
except NoSuchFormat:
continue
if opts.update_metadata:
with lopen(fmt_path, 'r+b') as stream:
update_metadata(mi, fmt, stream, plugboards, cdata)
return not formats_written, book_id, mi.title
def main(args=sys.argv):
parser = option_parser()
opts, args = parser.parse_args(args)
buf = BytesIO()
log = create_log(buf)
abort = Event()
authors = []
if opts.authors:
authors = string_to_authors(opts.authors)
identifiers = {}
if opts.isbn:
identifiers['isbn'] = opts.isbn
results = identify(log, abort, title=opts.title, authors=authors,
identifiers=identifiers, timeout=int(opts.timeout))
if not results:
print (log, file=sys.stderr)
prints('No results found', file=sys.stderr)
raise SystemExit(1)
result = results[0]
cf = None
if opts.cover and results:
cover = download_cover(log, title=opts.title, authors=authors,
identifiers=result.identifiers, timeout=int(opts.timeout))
if cover is None:
prints('No cover found', file=sys.stderr)
else:
save_cover_data_to(cover[-1], opts.cover)
result.cover = cf = opts.cover
log = buf.getvalue()
result = (metadata_to_opf(result) if opts.opf else
unicode(result).encode('utf-8'))
if opts.verbose:
print (log, file=sys.stderr)
print (result)
if not opts.opf and opts.cover:
prints('Cover :', cf)
return 0
def read_metadata_bulk(get_opf, get_cover, paths):
mi = metadata_from_formats(paths)
mi.cover = None
cdata = None
if mi.cover_data:
cdata = mi.cover_data[-1]
mi.cover_data = (None, None)
if not mi.application_id:
mi.application_id = '__calibre_dummy__'
ans = {'opf': None, 'cdata': None}
if get_opf:
ans['opf'] = metadata_to_opf(mi, default_lang='und')
if get_cover:
ans['cdata'] = cdata
return ans
def read_metadata_bulk(get_opf, get_cover, paths):
mi = metadata_from_formats(paths)
mi.cover = None
cdata = None
if mi.cover_data:
cdata = mi.cover_data[-1]
mi.cover_data = (None, None)
if not mi.application_id:
mi.application_id = '__calibre_dummy__'
ans = {'opf': None, 'cdata': None}
if get_opf:
ans['opf'] = metadata_to_opf(mi, default_lang='und')
if get_cover:
ans['cdata'] = cdata
return ans
def main(do_identify, covers, metadata, ensure_fields, tdir):
failed_ids = set()
failed_covers = set()
all_failed = True
log = GUILog()
patch_plugins()
for book_id, mi in metadata.iteritems():
mi = OPF(BytesIO(mi), basedir=tdir,
populate_spine=False).to_book_metadata()
title, authors, identifiers = mi.title, mi.authors, mi.identifiers
cdata = None
log.clear()
if do_identify:
results = []
try:
results = identify(log, Event(), title=title, authors=authors,
identifiers=identifiers)
except:
pass
if results:
all_failed = False
mi = merge_result(mi, results[0], ensure_fields=ensure_fields)
identifiers = mi.identifiers
if not mi.is_null('rating'):
# set_metadata expects a rating out of 10
mi.rating *= 2
with open(os.path.join(tdir, '%d.mi'%book_id), 'wb') as f:
f.write(metadata_to_opf(mi, default_lang='und'))
else:
log.error('Failed to download metadata for', title)
failed_ids.add(book_id)
if covers:
cdata = download_cover(log, title=title, authors=authors,
identifiers=identifiers)
if cdata is None:
failed_covers.add(book_id)
else:
with open(os.path.join(tdir, '%d.cover'%book_id), 'wb') as f:
f.write(cdata[-1])
all_failed = False
with open(os.path.join(tdir, '%d.log'%book_id), 'wb') as f:
f.write(log.plain_text.encode('utf-8'))
return failed_ids, failed_covers, all_failed
def serialize_metadata_for(paths, tdir, group_id):
mi = metadata_from_formats(paths)
mi.cover = None
cdata = None
if mi.cover_data:
cdata = mi.cover_data[-1]
mi.cover_data = (None, None)
if not mi.application_id:
mi.application_id = '__calibre_dummy__'
opf = metadata_to_opf(mi, default_lang='und')
has_cover = False
if cdata:
with open(os.path.join(tdir, '%s.cdata' % group_id), 'wb') as f:
f.write(cdata)
has_cover = True
return mi, opf, has_cover
def serialize_metadata_for(formats, tdir, id_):
from calibre.ebooks.metadata.meta import metadata_from_formats
from calibre.ebooks.metadata.opf2 import metadata_to_opf
mi = metadata_from_formats(formats)
mi.cover = None
cdata = None
if mi.cover_data:
cdata = mi.cover_data[-1]
mi.cover_data = None
if not mi.application_id:
mi.application_id = '__calibre_dummy__'
with open(os.path.join(tdir, '%s.opf' % id_), 'wb') as f:
f.write(metadata_to_opf(mi, default_lang='und'))
if cdata:
with open(os.path.join(tdir, str(id_)), 'wb') as f:
f.write(cdata)
def serialize_metadata_for(paths, tdir, group_id):
mi = metadata_from_formats(paths)
mi.cover = None
cdata = None
if mi.cover_data:
cdata = mi.cover_data[-1]
mi.cover_data = (None, None)
if not mi.application_id:
mi.application_id = '__calibre_dummy__'
opf = metadata_to_opf(mi, default_lang='und')
has_cover = False
if cdata:
with open(os.path.join(tdir, '%s.cdata' % group_id), 'wb') as f:
f.write(cdata)
has_cover = True
return mi, opf, has_cover
def serialize_metadata_for(formats, tdir, id_):
from calibre.ebooks.metadata.meta import metadata_from_formats
from calibre.ebooks.metadata.opf2 import metadata_to_opf
mi = metadata_from_formats(formats)
mi.cover = None
cdata = None
if mi.cover_data:
cdata = mi.cover_data[-1]
mi.cover_data = None
if not mi.application_id:
mi.application_id = '__calibre_dummy__'
with open(os.path.join(tdir, '%s.opf'%id_), 'wb') as f:
f.write(metadata_to_opf(mi, default_lang='und'))
if cdata:
with open(os.path.join(tdir, str(id_)), 'wb') as f:
f.write(cdata)
def main(argv):
if (len(argv) > 1 and argv[1] in ('--help', '-h')):
print usage(argv[0])
sys.exit(0)
if len(argv) > 2:
print >> sys.stderr, usage()
sys.exit(1)
dbdir = argv[1].rstrip(os.sep) if len(argv) > 1 else '.'
dbname = os.path.join(dbdir, 'metadata.db')
dbpath = os.path.abspath(dbname)
print >> sys.stderr, 'info: Reading Calibre database: ' + dbpath
if not os.path.isfile(dbname):
raise AssertionError('Calibre database missing: %s' % dbpath)
dbdirpath = os.path.dirname(dbpath)
# Reads the whole metadata.db to memory.
db = database2.LibraryDatabase2(dbdirpath)
fm = db.FIELD_MAP
fm_path = fm['path']
# TODO(pts): Do we have to replace / with os.sep on Windows?
book_path_items = [(i, encode_unicode(row[fm_path]))
for i, row in enumerate(db.data._data)
if row is not None]
print >> sys.stderr, 'info: Found %d book%s in metadata.db.' % (
len(book_path_items), 's' * (len(book_path_items) != 1))
book_opf_items = []
for i, book_path in book_path_items:
opf_path = os.path.join(dbdirpath, book_path, 'metadata.opf')
if not os.path.exists(opf_path):
book_opf_items.append((i, opf_path))
print >> sys.stderr, 'info: Found %d book%s with missing metadata.opf.' % (
len(book_opf_items), 's' * (len(book_opf_items) != 1))
# TODO(pts): Find directories not corresponding to any books in the database.
for i, opf_path in book_opf_items:
mi = db.get_metadata(i, index_is_id=True)
print >> sys.stderr, 'info: Creating metadata.opf: %s' % opf_path
# This creates a very different .opf.
# opf_creator = opf2.OPFCreator(os.path.dirname(opf_path), mi)
# sio = cStringIO.StringIO()
# opf_creator.render(sio)
if mi.has_cover and not mi.cover:
mi.cover = 'cover.jpg'
data = opf2.metadata_to_opf(mi)
with open(opf_path, 'wb') as f:
f.write(data)
print >> sys.stderr, 'info: metadata.opf creation done.'
def forked_read_metadata(path, tdir):
from calibre.ebooks.metadata.opf2 import metadata_to_opf
with lopen(path, 'rb') as f:
fmt = os.path.splitext(path)[1][1:].lower()
f.seek(0, 2)
sz = f.tell()
with lopen(os.path.join(tdir, 'size.txt'), 'wb') as s:
s.write(str(sz).encode('ascii'))
f.seek(0)
mi = get_metadata(f, fmt)
if mi.cover_data and mi.cover_data[1]:
with lopen(os.path.join(tdir, 'cover.jpg'), 'wb') as f:
f.write(mi.cover_data[1])
mi.cover_data = (None, None)
mi.cover = 'cover.jpg'
opf = metadata_to_opf(mi, default_lang='und')
with lopen(os.path.join(tdir, 'metadata.opf'), 'wb') as f:
f.write(opf)
def forked_read_metadata(path, tdir):
from calibre.ebooks.metadata.opf2 import metadata_to_opf
with lopen(path, 'rb') as f:
fmt = os.path.splitext(path)[1][1:].lower()
f.seek(0, 2)
sz = f.tell()
with lopen(os.path.join(tdir, 'size.txt'), 'wb') as s:
s.write(str(sz).encode('ascii'))
f.seek(0)
mi = get_metadata(f, fmt)
if mi.cover_data and mi.cover_data[1]:
with lopen(os.path.join(tdir, 'cover.jpg'), 'wb') as f:
f.write(mi.cover_data[1])
mi.cover_data = (None, None)
mi.cover = 'cover.jpg'
opf = metadata_to_opf(mi, default_lang='und')
with lopen(os.path.join(tdir, 'metadata.opf'), 'wb') as f:
f.write(opf)
def get(ctx, rd, what, book_id, library_id):
db = ctx.get_library(library_id)
if db is None:
raise HTTPNotFound("Library %r not found" % library_id)
with db.safe_read_lock:
if book_id not in ctx.allowed_book_ids(rd, db):
raise HTTPNotFound("Book with id %r does not exist" % book_id)
library_id = db.server_library_id # in case library_id was None
if what == "thumb":
sz = rd.query.get("sz")
w, h = 60, 80
if sz is None:
pass
elif sz == "full":
w = h = None
elif "x" in sz:
try:
w, h = map(int, sz.partition("x")[::2])
except Exception:
pass
else:
try:
w = h = int(sz)
except Exception:
pass
return cover(ctx, rd, library_id, db, book_id, width=w, height=h)
elif what == "cover":
return cover(ctx, rd, library_id, db, book_id)
elif what == "opf":
mi = db.get_metadata(book_id, get_cover=False)
rd.outheaders["Content-Type"] = "application/oebps-package+xml; charset=UTF-8"
rd.outheaders["Last-Modified"] = http_date(timestampfromdt(mi.last_modified))
return metadata_to_opf(mi)
elif what == "json":
from calibre.srv.ajax import book_to_json
data, last_modified = book_to_json(ctx, rd, db, book_id)
rd.outheaders["Last-Modified"] = http_date(timestampfromdt(last_modified))
return json(ctx, rd, get, data)
else:
try:
return book_fmt(ctx, rd, library_id, db, book_id, what.lower())
except NoSuchFormat:
raise HTTPNotFound("No %r format for the book %r" % (what.lower(), book_id))
def main(argv):
if (len(argv) > 1 and argv[1] in ('--help', '-h')):
print usage(argv[0])
sys.exit(0)
if len(argv) > 2:
print >>sys.stderr, usage()
sys.exit(1)
dbdir = argv[1].rstrip(os.sep) if len(argv) > 1 else '.'
dbname = os.path.join(dbdir, 'metadata.db')
dbpath = os.path.abspath(dbname)
print >>sys.stderr, 'info: Reading Calibre database: ' + dbpath
if not os.path.isfile(dbname):
raise AssertionError('Calibre database missing: %s' % dbpath)
dbdirpath = os.path.dirname(dbpath)
# Reads the whole metadata.db to memory.
db = database2.LibraryDatabase2(dbdirpath)
fm = db.FIELD_MAP
fm_path = fm['path']
# TODO(pts): Do we have to replace / with os.sep on Windows?
book_path_items = [(i, encode_unicode(row[fm_path]))
for i, row in enumerate(db.data._data) if row is not None]
print >>sys.stderr, 'info: Found %d book%s in metadata.db.' % (
len(book_path_items), 's' * (len(book_path_items) != 1))
book_opf_items = []
for i, book_path in book_path_items:
opf_path = os.path.join(dbdirpath, book_path, 'metadata.opf')
if not os.path.exists(opf_path):
book_opf_items.append((i, opf_path))
print >>sys.stderr, 'info: Found %d book%s with missing metadata.opf.' % (
len(book_opf_items), 's' * (len(book_opf_items) != 1))
# TODO(pts): Find directories not corresponding to any books in the database.
for i, opf_path in book_opf_items:
mi = db.get_metadata(i, index_is_id=True)
print >>sys.stderr, 'info: Creating metadata.opf: %s' % opf_path
# This creates a very different .opf.
# opf_creator = opf2.OPFCreator(os.path.dirname(opf_path), mi)
# sio = cStringIO.StringIO()
# opf_creator.render(sio)
if mi.has_cover and not mi.cover:
mi.cover = 'cover.jpg'
data = opf2.metadata_to_opf(mi)
with open(opf_path, 'wb') as f:
f.write(data)
print >>sys.stderr, 'info: metadata.opf creation done.'
def copy_metadata(self):
rows = self.gui.library_view.selectionModel().selectedRows()
if not rows or len(rows) == 0:
return error_dialog(self.gui, _('Cannot copy metadata'),
_('No books selected'), show=True)
if len(rows) > 1:
return error_dialog(self.gui, _('Cannot copy metadata'),
_('Multiple books selected, can only copy from one book at a time.'), show=True)
db = self.gui.current_db
book_id = db.id(rows[0].row())
mi = db.new_api.get_metadata(book_id)
md = QMimeData()
md.setText(unicode(mi))
md.setData('application/calibre-book-metadata', bytearray(metadata_to_opf(mi, default_lang='und')))
img = db.new_api.cover(book_id, as_image=True)
if img:
md.setImageData(img)
c = QApplication.clipboard()
c.setMimeData(md)
def forked_read_metadata(path, tdir):
from calibre.ebooks.metadata.opf2 import metadata_to_opf
with open(path, "rb") as f:
fmt = os.path.splitext(path)[1][1:].lower()
f.seek(0, 2)
sz = f.tell()
with open(os.path.join(tdir, "size.txt"), "wb") as s:
s.write(str(sz).encode("ascii"))
f.seek(0)
mi = get_metadata(f, fmt)
if mi.cover_data and mi.cover_data[1]:
with open(os.path.join(tdir, "cover.jpg"), "wb") as f:
f.write(mi.cover_data[1])
mi.cover_data = (None, None)
mi.cover = "cover.jpg"
opf = metadata_to_opf(mi, default_lang="und")
with open(os.path.join(tdir, "metadata.opf"), "wb") as f:
f.write(opf)
def commit(self, book_id):
''' Commits any local changes to the metadata for this book up to the server '''
ids = self.get_arg_ids()
if book_id not in ids:
print('There is no a*rg identifier for this book')
return
arg_id, timestamp, version = ids[book_id]
repo_version = self.arg_book_status(arg_id)
if version!=repo_version:
print('This library\'s version is out of date.')
return
# recompute the id
mi = get_gui().current_db.get_metadata(book_id, index_is_id=True, get_cover=True)
mi.identifiers['arg'] = "%s.%s" % (mi.identifiers['arg'].split('.')[0], version+1)
get_gui().current_db.set_metadata(book_id, mi)
mi = get_gui().current_db.get_metadata(book_id, index_is_id=True, get_cover=True)
# get the opf that will be posted
opf = metadata_to_opf(mi)
# now post the metadata
return self._post_metadata(arg_id, version, opf, mi.cover)
def forked_read_metadata(original_path, tdir):
from calibre.ebooks.metadata.opf2 import metadata_to_opf
from calibre.ebooks.metadata.worker import run_import_plugins
path = run_import_plugins((original_path, ), os.getpid(), tdir)[0]
if path != original_path:
with lopen(os.path.join(tdir, 'file_changed_by_plugins'), 'w') as f:
f.write(os.path.abspath(path))
with lopen(path, 'rb') as f:
fmt = os.path.splitext(path)[1][1:].lower()
f.seek(0, 2)
sz = f.tell()
with lopen(os.path.join(tdir, 'size.txt'), 'wb') as s:
s.write(str(sz).encode('ascii'))
f.seek(0)
mi = get_metadata(f, fmt)
if mi.cover_data and mi.cover_data[1]:
with lopen(os.path.join(tdir, 'cover.jpg'), 'wb') as f:
f.write(mi.cover_data[1])
mi.cover_data = (None, None)
mi.cover = 'cover.jpg'
opf = metadata_to_opf(mi, default_lang='und')
with lopen(os.path.join(tdir, 'metadata.opf'), 'wb') as f:
f.write(opf)
def run(self):
while self.keep_running:
try:
time.sleep(2) # Limit to one book per two seconds
(id_, sequence) = self.db.get_a_dirtied_book()
if id_ is None:
continue
# print 'writer thread', id_, sequence
except:
# Happens during interpreter shutdown
break
if not self.keep_running:
break
try:
path, mi, sequence = self.get_metadata_for_dump(id_)
except:
prints('Failed to get backup metadata for id:', id_, 'once')
traceback.print_exc()
time.sleep(2)
try:
path, mi, sequence = self.get_metadata_for_dump(id_)
except:
prints('Failed to get backup metadata for id:', id_, 'again, giving up')
traceback.print_exc()
continue
if mi is None:
self.clear_dirtied(id_, sequence)
continue
if not self.keep_running:
break
# Give the GUI thread a chance to do something. Python threads don't
# have priorities, so this thread would naturally keep the processor
# until some scheduling event happens. The sleep makes such an event
time.sleep(0.1)
try:
raw = metadata_to_opf(mi)
except:
prints('Failed to convert to opf for id:', id_)
traceback.print_exc()
continue
if not self.keep_running:
break
time.sleep(0.1) # Give the GUI thread a chance to do something
try:
self.do_write(path, raw)
except:
prints('Failed to write backup metadata for id:', id_, 'once')
time.sleep(2)
try:
self.do_write(path, raw)
except:
prints('Failed to write backup metadata for id:', id_,
'again, giving up')
continue
self.clear_dirtied(id_, sequence)
self.break_cycles()
def apply_downloaded_metadata(self, review, payload, *args):
good_ids, tdir, log_file, lm_map, failed_ids = payload
if not good_ids:
return
restrict_to_failed = False
modified = set()
db = self.gui.current_db
for i in good_ids:
lm = db.metadata_last_modified(i, index_is_id=True)
if lm is not None and lm_map[i] is not None and lm > lm_map[i]:
title = db.title(i, index_is_id=True)
authors = db.authors(i, index_is_id=True)
if authors:
authors = [x.replace('|', ',') for x in authors.split(',')]
title += ' - ' + authors_to_string(authors)
modified.add(title)
if modified:
from calibre.utils.icu import lower
modified = sorted(modified, key=lower)
if not question_dialog(
self.gui,
_('Some books changed'),
'<p>' +
_('The metadata for some books in your library has'
' changed since you started the download. If you'
' proceed, some of those changes may be overwritten. '
'Click "Show details" to see the list of changed books. '
'Do you want to proceed?'),
det_msg='\n'.join(modified)):
return
id_map = {}
for bid in good_ids:
opf = os.path.join(tdir, '%d.mi' % bid)
if not os.path.exists(opf):
opf = None
cov = os.path.join(tdir, '%d.cover' % bid)
if not os.path.exists(cov):
cov = None
id_map[bid] = (opf, cov)
if review:
def get_metadata(book_id):
oldmi = db.get_metadata(book_id,
index_is_id=True,
get_cover=True,
cover_as_data=True)
opf, cov = id_map[book_id]
if opf is None:
newmi = Metadata(oldmi.title, authors=tuple(oldmi.authors))
else:
with open(opf, 'rb') as f:
newmi = OPF(f,
basedir=os.path.dirname(opf),
populate_spine=False).to_book_metadata()
newmi.cover, newmi.cover_data = None, (None, None)
for x in ('title', 'authors'):
if newmi.is_null(x):
# Title and author are set to null if they are
# the same as the originals as an optimization,
# we undo that, as it is confusing.
newmi.set(x, copy.copy(oldmi.get(x)))
if cov:
with open(cov, 'rb') as f:
newmi.cover_data = ('jpg', f.read())
return oldmi, newmi
from calibre.gui2.metadata.diff import CompareMany
d = CompareMany(
set(id_map),
get_metadata,
db.field_metadata,
parent=self.gui,
window_title=_('Review downloaded metadata'),
reject_button_tooltip=_(
'Discard downloaded metadata for this book'),
accept_all_tooltip=_(
'Use the downloaded metadata for all remaining books'),
reject_all_tooltip=_(
'Discard downloaded metadata for all remaining books'),
revert_tooltip=_('Discard the downloaded value for: %s'),
intro_msg=
_('The downloaded metadata is on the left and the original metadata'
' is on the right. If a downloaded value is blank or unknown,'
' the original value is used.'),
action_button=(_('&View book'), I('view.png'),
self.gui.iactions['View'].view_historical),
db=db)
if d.exec() == QDialog.DialogCode.Accepted:
if d.mark_rejected:
failed_ids |= d.rejected_ids
restrict_to_failed = True
nid_map = {}
for book_id, (changed, mi) in iteritems(d.accepted):
if mi is None: # discarded
continue
if changed:
opf, cov = id_map[book_id]
cfile = mi.cover
mi.cover, mi.cover_data = None, (None, None)
if opf is not None:
with open(opf, 'wb') as f:
f.write(metadata_to_opf(mi))
if cfile and cov:
shutil.copyfile(cfile, cov)
os.remove(cfile)
nid_map[book_id] = id_map[book_id]
id_map = nid_map
else:
id_map = {}
restrict_to_failed = restrict_to_failed or bool(args and args[0])
restrict_to_failed = restrict_to_failed and bool(failed_ids)
if restrict_to_failed:
db.data.set_marked_ids(failed_ids)
self.apply_metadata_changes(id_map,
merge_comments=msprefs['append_comments'],
icon='download-metadata.png',
callback=partial(
self.downloaded_metadata_applied, tdir,
restrict_to_failed))
def apply_downloaded_metadata(self, review, payload, *args):
good_ids, tdir, log_file, lm_map, failed_ids = payload
if not good_ids:
return
restrict_to_failed = False
modified = set()
db = self.gui.current_db
for i in good_ids:
lm = db.metadata_last_modified(i, index_is_id=True)
if lm is not None and lm_map[i] is not None and lm > lm_map[i]:
title = db.title(i, index_is_id=True)
authors = db.authors(i, index_is_id=True)
if authors:
authors = [x.replace('|', ',') for x in authors.split(',')]
title += ' - ' + authors_to_string(authors)
modified.add(title)
if modified:
from calibre.utils.icu import lower
modified = sorted(modified, key=lower)
if not question_dialog(self.gui, _('Some books changed'), '<p>' + _(
'The metadata for some books in your library has'
' changed since you started the download. If you'
' proceed, some of those changes may be overwritten. '
'Click "Show details" to see the list of changed books. '
'Do you want to proceed?'), det_msg='\n'.join(modified)):
return
id_map = {}
for bid in good_ids:
opf = os.path.join(tdir, '%d.mi'%bid)
if not os.path.exists(opf):
opf = None
cov = os.path.join(tdir, '%d.cover'%bid)
if not os.path.exists(cov):
cov = None
id_map[bid] = (opf, cov)
if review:
def get_metadata(book_id):
oldmi = db.get_metadata(book_id, index_is_id=True, get_cover=True, cover_as_data=True)
opf, cov = id_map[book_id]
if opf is None:
newmi = Metadata(oldmi.title, authors=tuple(oldmi.authors))
else:
with open(opf, 'rb') as f:
newmi = OPF(f, basedir=os.path.dirname(opf), populate_spine=False).to_book_metadata()
newmi.cover, newmi.cover_data = None, (None, None)
for x in ('title', 'authors'):
if newmi.is_null(x):
# Title and author are set to null if they are
# the same as the originals as an optimization,
# we undo that, as it is confusing.
newmi.set(x, copy.copy(oldmi.get(x)))
if cov:
with open(cov, 'rb') as f:
newmi.cover_data = ('jpg', f.read())
return oldmi, newmi
from calibre.gui2.metadata.diff import CompareMany
d = CompareMany(
set(id_map), get_metadata, db.field_metadata, parent=self.gui,
window_title=_('Review downloaded metadata'),
reject_button_tooltip=_('Discard downloaded metadata for this book'),
accept_all_tooltip=_('Use the downloaded metadata for all remaining books'),
reject_all_tooltip=_('Discard downloaded metadata for all remaining books'),
revert_tooltip=_('Discard the downloaded value for: %s'),
intro_msg=_('The downloaded metadata is on the left and the original metadata'
' is on the right. If a downloaded value is blank or unknown,'
' the original value is used.'),
action_button=(_('&View Book'), I('view.png'), self.gui.iactions['View'].view_historical),
db=db
)
if d.exec_() == d.Accepted:
if d.mark_rejected:
failed_ids |= d.rejected_ids
restrict_to_failed = True
nid_map = {}
for book_id, (changed, mi) in d.accepted.iteritems():
if mi is None: # discarded
continue
if changed:
opf, cov = id_map[book_id]
cfile = mi.cover
mi.cover, mi.cover_data = None, (None, None)
if opf is not None:
with open(opf, 'wb') as f:
f.write(metadata_to_opf(mi))
if cfile and cov:
shutil.copyfile(cfile, cov)
os.remove(cfile)
nid_map[book_id] = id_map[book_id]
id_map = nid_map
else:
id_map = {}
restrict_to_failed = restrict_to_failed or bool(args and args[0])
restrict_to_failed = restrict_to_failed and bool(failed_ids)
if restrict_to_failed:
db.data.set_marked_ids(failed_ids)
self.apply_metadata_changes(
id_map, merge_comments=msprefs['append_comments'], icon='download-metadata.png',
callback=partial(self.downloaded_metadata_applied, tdir, restrict_to_failed))
def download(all_ids,
tf,
db,
do_identify,
covers,
ensure_fields,
log=None,
abort=None,
notifications=None):
batch_size = 10
batches = split_jobs(all_ids, batch_size=batch_size)
tdir = PersistentTemporaryDirectory('_metadata_bulk')
heartbeat = HeartBeat(tdir)
failed_ids = set()
failed_covers = set()
title_map = {}
lm_map = {}
ans = set()
all_failed = True
aborted = False
count = 0
notifier = Notifier(notifications, title_map, tdir, len(all_ids))
notifier.start()
try:
for ids in batches:
if abort.is_set():
log.error('Aborting...')
break
metadata = {
i: db.get_metadata(i,
index_is_id=True,
get_user_categories=False)
for i in ids
}
for i in ids:
title_map[i] = metadata[i].title
lm_map[i] = metadata[i].last_modified
metadata = {
i: metadata_to_opf(mi, default_lang='und')
for i, mi in iteritems(metadata)
}
try:
ret = fork_job(
'calibre.ebooks.metadata.sources.worker',
'main',
(do_identify, covers, metadata, ensure_fields, tdir),
abort=abort,
heartbeat=heartbeat,
no_output=True)
except WorkerError as e:
if e.orig_tb:
raise Exception('Failed to download metadata. Original '
'traceback: \n\n' + e.orig_tb)
raise
count += batch_size
fids, fcovs, allf = ret['result']
if not allf:
all_failed = False
failed_ids = failed_ids.union(fids)
failed_covers = failed_covers.union(fcovs)
ans = ans.union(set(ids) - fids)
for book_id in ids:
lp = os.path.join(tdir, '%d.log' % book_id)
if os.path.exists(lp):
with open(tf, 'ab') as dest, open(lp, 'rb') as src:
dest.write(('\n' + '#' * 20 +
' Log for %s ' % title_map[book_id] +
'#' * 20 + '\n').encode('utf-8'))
shutil.copyfileobj(src, dest)
if abort.is_set():
aborted = True
log('Download complete, with %d failures' % len(failed_ids))
return (aborted, ans, tdir, tf, failed_ids, failed_covers, title_map,
lm_map, all_failed)
finally:
notifier.keep_going = False
def convert(self, oeb_book, output_path, input_plugin, opts, log):
from lxml import etree
from calibre.ebooks.oeb.base import OEB_IMAGES, SVG_MIME
from calibre.ebooks.metadata.opf2 import OPF, metadata_to_opf
from calibre.utils.zipfile import ZipFile
from calibre.utils.filenames import ascii_filename
# HTML
if opts.htmlz_css_type == 'inline':
from calibre.ebooks.htmlz.oeb2html import OEB2HTMLInlineCSSizer
OEB2HTMLizer = OEB2HTMLInlineCSSizer
elif opts.htmlz_css_type == 'tag':
from calibre.ebooks.htmlz.oeb2html import OEB2HTMLNoCSSizer
OEB2HTMLizer = OEB2HTMLNoCSSizer
else:
from calibre.ebooks.htmlz.oeb2html import OEB2HTMLClassCSSizer as OEB2HTMLizer
with TemporaryDirectory(u'_htmlz_output') as tdir:
htmlizer = OEB2HTMLizer(log)
html = htmlizer.oeb2html(oeb_book, opts)
fname = u'index'
if opts.htmlz_title_filename:
from calibre.utils.filenames import shorten_components_to
fname = shorten_components_to(100, (ascii_filename(
unicode_type(oeb_book.metadata.title[0])), ))[0]
with open(os.path.join(tdir, fname + u'.html'), 'wb') as tf:
if isinstance(html, unicode_type):
html = html.encode('utf-8')
tf.write(html)
# CSS
if opts.htmlz_css_type == 'class' and opts.htmlz_class_style == 'external':
with open(os.path.join(tdir, u'style.css'), 'wb') as tf:
tf.write(htmlizer.get_css(oeb_book))
# Images
images = htmlizer.images
if images:
if not os.path.exists(os.path.join(tdir, u'images')):
os.makedirs(os.path.join(tdir, u'images'))
for item in oeb_book.manifest:
if item.media_type in OEB_IMAGES and item.href in images:
if item.media_type == SVG_MIME:
data = unicode_type(
etree.tostring(item.data,
encoding=unicode_type))
else:
data = item.data
fname = os.path.join(tdir, u'images',
images[item.href])
with open(fname, 'wb') as img:
img.write(data)
# Cover
cover_path = None
try:
cover_data = None
if oeb_book.metadata.cover:
term = oeb_book.metadata.cover[0].term
cover_data = oeb_book.guide[term].item.data
if cover_data:
from calibre.utils.img import save_cover_data_to
cover_path = os.path.join(tdir, u'cover.jpg')
with lopen(cover_path, 'w') as cf:
cf.write('')
save_cover_data_to(cover_data, cover_path)
except:
import traceback
traceback.print_exc()
# Metadata
with open(os.path.join(tdir, u'metadata.opf'), 'wb') as mdataf:
opf = OPF(
io.BytesIO(
etree.tostring(oeb_book.metadata.to_opf1(),
encoding='UTF-8')))
mi = opf.to_book_metadata()
if cover_path:
mi.cover = u'cover.jpg'
mdataf.write(metadata_to_opf(mi))
htmlz = ZipFile(output_path, 'w')
htmlz.add_dir(tdir)
def create_book(mi,
path,
fmt='epub',
opf_name='metadata.opf',
html_name='start.xhtml',
toc_name='toc.ncx'):
''' Create an empty book in the specified format at the specified location. '''
if fmt not in valid_empty_formats:
raise ValueError('Cannot create empty book in the %s format' % fmt)
if fmt == 'txt':
with open(path, 'wb') as f:
if not mi.is_null('title'):
f.write(as_bytes(mi.title))
return
if fmt == 'docx':
from calibre.ebooks.conversion.plumber import Plumber
from calibre.ebooks.docx.writer.container import DOCX
from calibre.utils.logging import default_log
p = Plumber('a.docx', 'b.docx', default_log)
p.setup_options()
# Use the word default of one inch page margins
for x in 'left right top bottom'.split():
setattr(p.opts, 'margin_' + x, 72)
DOCX(p.opts, default_log).write(path, mi, create_empty_document=True)
return
path = os.path.abspath(path)
lang = 'und'
opf = metadata_to_opf(mi, as_string=False)
for l in opf.xpath('//*[local-name()="language"]'):
if l.text:
lang = l.text
break
lang = lang_as_iso639_1(lang) or lang
opfns = OPF_NAMESPACES['opf']
m = opf.makeelement('{%s}manifest' % opfns)
opf.insert(1, m)
i = m.makeelement('{%s}item' % opfns, href=html_name, id='start')
i.set('media-type', guess_type('a.xhtml'))
m.append(i)
i = m.makeelement('{%s}item' % opfns, href=toc_name, id='ncx')
i.set('media-type', guess_type(toc_name))
m.append(i)
s = opf.makeelement('{%s}spine' % opfns, toc="ncx")
opf.insert(2, s)
i = s.makeelement('{%s}itemref' % opfns, idref='start')
s.append(i)
CONTAINER = '''\
<?xml version="1.0"?>
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
<rootfiles>
<rootfile full-path="{0}" media-type="application/oebps-package+xml"/>
</rootfiles>
</container>
'''.format(prepare_string_for_xml(opf_name, True)).encode('utf-8')
HTML = P('templates/new_book.html', data=True).decode('utf-8').replace(
'_LANGUAGE_', prepare_string_for_xml(lang, True)).replace(
'_TITLE_', prepare_string_for_xml(mi.title)).replace(
'_AUTHORS_',
prepare_string_for_xml(authors_to_string(
mi.authors))).encode('utf-8')
h = parse(HTML)
pretty_html_tree(None, h)
HTML = serialize(h, 'text/html')
ncx = etree.tostring(create_toc(mi, opf, html_name, lang),
encoding='utf-8',
xml_declaration=True,
pretty_print=True)
pretty_xml_tree(opf)
opf = etree.tostring(opf,
encoding='utf-8',
xml_declaration=True,
pretty_print=True)
if fmt == 'azw3':
with TemporaryDirectory('create-azw3') as tdir, CurrentDir(tdir):
for name, data in ((opf_name, opf), (html_name, HTML), (toc_name,
ncx)):
with open(name, 'wb') as f:
f.write(data)
c = Container(os.path.dirname(os.path.abspath(opf_name)), opf_name,
DevNull())
opf_to_azw3(opf_name, path, c)
else:
with ZipFile(path, 'w', compression=ZIP_STORED) as zf:
zf.writestr('mimetype',
b'application/epub+zip',
compression=ZIP_STORED)
zf.writestr('META-INF/', b'', 0o755)
zf.writestr('META-INF/container.xml', CONTAINER)
zf.writestr(opf_name, opf)
zf.writestr(html_name, HTML)
zf.writestr(toc_name, ncx)
def do_save_book_to_disk(id_, mi, cover, plugboards,
format_map, root, opts, length):
available_formats = [x.lower().strip() for x in format_map.keys()]
if mi.pubdate:
mi.pubdate = as_local_time(mi.pubdate)
if mi.timestamp:
mi.timestamp = as_local_time(mi.timestamp)
if opts.formats == 'all':
asked_formats = available_formats
else:
asked_formats = [x.lower().strip() for x in opts.formats.split(',')]
formats = set(available_formats).intersection(set(asked_formats))
if not formats:
return True, id_, mi.title
components = get_path_components(opts, mi, id_, length)
base_path = os.path.join(root, *components)
base_name = os.path.basename(base_path)
dirpath = os.path.dirname(base_path)
# Don't test for existence first as the test could fail but
# another worker process could create the directory before
# the call to makedirs
try:
os.makedirs(dirpath)
except BaseException:
if not os.path.exists(dirpath):
raise
ocover = mi.cover
if opts.save_cover and cover:
with open(base_path+'.jpg', 'wb') as f:
f.write(cover)
mi.cover = base_name+'.jpg'
else:
mi.cover = None
if opts.write_opf:
from calibre.ebooks.metadata.opf2 import metadata_to_opf
opf = metadata_to_opf(mi)
with open(base_path+'.opf', 'wb') as f:
f.write(opf)
mi.cover = ocover
written = False
for fmt in formats:
fp = format_map.get(fmt, None)
if fp is None:
continue
stream = SpooledTemporaryFile(20*1024*1024, '_save_to_disk.'+(fmt or
'tmp'))
with open(fp, 'rb') as f:
shutil.copyfileobj(f, stream)
stream.seek(0)
written = True
if opts.update_metadata:
update_metadata(mi, fmt, stream, plugboards, cover)
stream.seek(0)
fmt_path = base_path+'.'+str(fmt)
with open(fmt_path, 'wb') as f:
shutil.copyfileobj(stream, f)
return not written, id_, mi.title
def auto_add(self):
from calibre.utils.ipc.simple_worker import fork_job, WorkerError
from calibre.ebooks.metadata.opf2 import metadata_to_opf
from calibre.ebooks.metadata.meta import metadata_from_filename
files = [x for x in os.listdir(self.path) if
# Must not be in the process of being added to the db
x not in self.staging
# Firefox creates 0 byte placeholder files when downloading
and os.stat(os.path.join(self.path, x)).st_size > 0
# Must be a file
and os.path.isfile(os.path.join(self.path, x))
# Must have read and write permissions
and os.access(os.path.join(self.path, x), os.R_OK|os.W_OK)
# Must be a known ebook file type
and os.path.splitext(x)[1][1:].lower() in self.allowed
]
data = {}
# Give any in progress copies time to complete
time.sleep(2)
for fname in files:
f = os.path.join(self.path, fname)
# Try opening the file for reading, if the OS prevents us, then at
# least on windows, it means the file is open in another
# application for writing. We will get notified by
# QFileSystemWatcher when writing is completed, so ignore for now.
try:
open(f, 'rb').close()
except:
continue
tdir = tempfile.mkdtemp(dir=self.tdir)
try:
fork_job('calibre.ebooks.metadata.meta',
'forked_read_metadata', (f, tdir), no_output=True)
except WorkerError as e:
prints('Failed to read metadata from:', fname)
prints(e.orig_tb)
except:
import traceback
traceback.print_exc()
# Ensure that the pre-metadata file size is present. If it isn't,
# write 0 so that the file is rescanned
szpath = os.path.join(tdir, 'size.txt')
try:
with open(szpath, 'rb') as f:
int(f.read())
except:
with open(szpath, 'wb') as f:
f.write(b'0')
opfpath = os.path.join(tdir, 'metadata.opf')
try:
if os.stat(opfpath).st_size < 30:
raise Exception('metadata reading failed')
except:
mi = metadata_from_filename(fname)
with open(opfpath, 'wb') as f:
f.write(metadata_to_opf(mi))
self.staging.add(fname)
data[fname] = tdir
if data:
self.callback(data)
try:
os.makedirs(dirpath)
except BaseException:
if not os.path.exists(dirpath):
raise
ocover = mi.cover
if opts.save_cover and cover:
with open(base_path+'.jpg', 'wb') as f:
f.write(cover)
mi.cover = base_name+'.jpg'
else:
mi.cover = None
if opts.write_opf:
opf = metadata_to_opf(mi)
with open(base_path+'.opf', 'wb') as f:
f.write(opf)
mi.cover = ocover
written = False
for fmt in formats:
global plugboard_save_to_disk_value, plugboard_any_format_value
cpb = find_plugboard(plugboard_save_to_disk_value, fmt, plugboards)
fp = format_map.get(fmt, None)
if fp is None:
continue
stream = SpooledTemporaryFile(20*1024*1024, '_save_to_disk.'+(fmt or
'tmp'))
with open(fp, 'rb') as f:
def auto_add(self):
from calibre.utils.ipc.simple_worker import fork_job, WorkerError
from calibre.ebooks.metadata.opf2 import metadata_to_opf
from calibre.ebooks.metadata.meta import metadata_from_filename
files = [x for x in os.listdir(self.path) if
# Must not be in the process of being added to the db
x not in self.staging and
# Firefox creates 0 byte placeholder files when downloading
os.stat(os.path.join(self.path, x)).st_size > 0 and
# Must be a file
os.path.isfile(os.path.join(self.path, x)) and
# Must have read and write permissions
os.access(os.path.join(self.path, x), os.R_OK|os.W_OK) and
# Must be a known ebook file type
self.is_filename_allowed(x)
]
data = {}
# Give any in progress copies time to complete
time.sleep(2)
for fname in files:
f = os.path.join(self.path, fname)
# Try opening the file for reading, if the OS prevents us, then at
# least on windows, it means the file is open in another
# application for writing. We will get notified by
# QFileSystemWatcher when writing is completed, so ignore for now.
try:
open(f, 'rb').close()
except:
continue
tdir = tempfile.mkdtemp(dir=self.tdir)
try:
fork_job('calibre.ebooks.metadata.meta',
'forked_read_metadata', (f, tdir), no_output=True)
except WorkerError as e:
prints('Failed to read metadata from:', fname)
prints(e.orig_tb)
except:
import traceback
traceback.print_exc()
# Ensure that the pre-metadata file size is present. If it isn't,
# write 0 so that the file is rescanned
szpath = os.path.join(tdir, 'size.txt')
try:
with open(szpath, 'rb') as f:
int(f.read())
except:
with open(szpath, 'wb') as f:
f.write(b'0')
opfpath = os.path.join(tdir, 'metadata.opf')
try:
if os.stat(opfpath).st_size < 30:
raise Exception('metadata reading failed')
except:
mi = metadata_from_filename(fname)
with open(opfpath, 'wb') as f:
f.write(metadata_to_opf(mi))
self.staging.add(fname)
data[fname] = tdir
if data:
self.callback(data)