def run(self):
try:
basedir = os.path.dirname(self.src_library_path)
try:
tdir = TemporaryDirectory('_rlib', dir=basedir)
tdir.__enter__()
except EnvironmentError:
# Incase we dont have permissions to create directories in the
# parent folder of the src library
tdir = TemporaryDirectory('_rlib')
with tdir as tdir:
self.library_path = tdir
self.scan_library()
if not self.load_preferences():
# Something went wrong with preferences restore. Start over
# with a new database and attempt to rebuild the structure
# from the metadata in the opf
dbpath = os.path.join(self.library_path, 'metadata.db')
if os.path.exists(dbpath):
os.remove(dbpath)
self.create_cc_metadata()
self.restore_books()
if self.successes == 0 and len(self.dirs) > 0:
raise Exception(('Something bad happened'))
self.replace_db()
except:
self.tb = traceback.format_exc()
def __enter__(self, *args):
'''
Add this plugin to the python path so that it's contents become directly importable.
Useful when bundling large python libraries into the plugin. Use it like this::
with plugin:
import something
'''
if self.plugin_path is not None:
from calibre.utils.zipfile import ZipFile
zf = ZipFile(self.plugin_path)
extensions = set([x.rpartition('.')[-1].lower() for x in
zf.namelist()])
zip_safe = True
for ext in ('pyd', 'so', 'dll', 'dylib'):
if ext in extensions:
zip_safe = False
break
if zip_safe:
sys.path.insert(0, self.plugin_path)
self.sys_insertion_path = self.plugin_path
else:
from calibre.ptempfile import TemporaryDirectory
self._sys_insertion_tdir = TemporaryDirectory('plugin_unzip')
self.sys_insertion_path = self._sys_insertion_tdir.__enter__(*args)
zf.extractall(self.sys_insertion_path)
sys.path.insert(0, self.sys_insertion_path)
zf.close()
def update_rapydscript():
from calibre_lzma.xz import compress
d = os.path.dirname
base = d(d(d(d(d(abspath(__file__))))))
base = os.path.join(base, 'rapydscript')
with TemporaryDirectory() as tdir:
subprocess.check_call([
'node', '--harmony',
os.path.join(base, 'bin', 'web-repl-export'), tdir
])
with open(os.path.join(tdir, 'rapydscript.js'), 'rb') as f:
raw = f.read()
path = P(COMPILER_PATH, allow_user_override=False)
with open(path, 'wb') as f:
compress(raw, f, 9)
def convert(self, stream, options, file_ext, log, accelerators):
from calibre.ebooks.chm.metadata import get_metadata_from_reader
from calibre.customize.ui import plugin_for_input_format
self.opts = options
log.debug('Processing CHM...')
with TemporaryDirectory('_chm2oeb') as tdir:
if not isinstance(tdir, unicode):
tdir = tdir.decode(filesystem_encoding)
html_input = plugin_for_input_format('html')
for opt in html_input.options:
setattr(options, opt.option.name, opt.recommended_value)
no_images = False # options.no_images
chm_name = stream.name
# chm_data = stream.read()
# closing stream so CHM can be opened by external library
stream.close()
log.debug('tdir=%s' % tdir)
log.debug('stream.name=%s' % stream.name)
debug_dump = False
odi = options.debug_pipeline
if odi:
debug_dump = os.path.join(odi, 'input')
mainname = self._chmtohtml(tdir, chm_name, no_images, log,
debug_dump=debug_dump)
mainpath = os.path.join(tdir, mainname)
metadata = get_metadata_from_reader(self._chm_reader)
encoding = self._chm_reader.get_encoding() or options.input_encoding or 'cp1252'
self._chm_reader.CloseCHM()
# print tdir, mainpath
# from calibre import ipython
# ipython()
options.debug_pipeline = None
options.input_encoding = 'utf-8'
uenc = encoding
if os.path.abspath(mainpath) in self._chm_reader.re_encoded_files:
uenc = 'utf-8'
htmlpath, toc = self._create_html_root(mainpath, log, uenc)
oeb = self._create_oebbook_html(htmlpath, tdir, options, log, metadata)
options.debug_pipeline = odi
if toc.count() > 1:
oeb.toc = self.parse_html_toc(oeb.spine[0])
oeb.manifest.remove(oeb.spine[0])
oeb.auto_generated_toc = False
return oeb
def convert(self, stream, options, file_ext, log, accelerators):
from calibre.ebooks.chm.metadata import get_metadata_from_reader
from calibre.customize.ui import plugin_for_input_format
self.opts = options
log.debug('Processing CHM...')
with TemporaryDirectory('_chm2oeb') as tdir:
if not isinstance(tdir, unicode):
tdir = tdir.decode(filesystem_encoding)
html_input = plugin_for_input_format('html')
for opt in html_input.options:
setattr(options, opt.option.name, opt.recommended_value)
no_images = False #options.no_images
chm_name = stream.name
#chm_data = stream.read()
#closing stream so CHM can be opened by external library
stream.close()
log.debug('tdir=%s' % tdir)
log.debug('stream.name=%s' % stream.name)
debug_dump = False
odi = options.debug_pipeline
if odi:
debug_dump = os.path.join(odi, 'input')
mainname = self._chmtohtml(tdir,
chm_name,
no_images,
log,
debug_dump=debug_dump)
mainpath = os.path.join(tdir, mainname)
metadata = get_metadata_from_reader(self._chm_reader)
self._chm_reader.CloseCHM()
#print tdir
#from calibre import ipython
#ipython()
options.debug_pipeline = None
options.input_encoding = 'utf-8'
# try a custom conversion:
#oeb = self._create_oebbook(mainpath, tdir, options, log, metadata)
# try using html converter:
htmlpath = self._create_html_root(mainpath, log)
oeb = self._create_oebbook_html(htmlpath, tdir, options, log,
metadata)
options.debug_pipeline = odi
#log.debug('DEBUG: Not removing tempdir %s' % tdir)
return oeb
def convert(self, stream: IO, options, file_ext, log, accelerators):
from calibre.ebooks.oeb.base import DirContainer
from calibre.ebooks.conversion.plumber import create_oebbook
log.debug("Parsing UMD file...")
book = UMDFile.from_stream(stream)
log.debug("Handle meta data ...")
oeb = create_oebbook(log,
None,
options,
encoding=options.input_encoding,
populate=False)
oeb.metadata.add('title', book.title)
oeb.metadata.add('creator', book.author, attrib={'role': 'aut'})
oeb.metadata.add('publisher', book.publisher)
oeb.metadata.add('identifier',
str(uuid.uuid4()),
id='uuid_id',
scheme='uuid')
for id_ in oeb.metadata.identifier:
if 'id' in id_.attrib:
oeb.uid = oeb.metadata.identifier[0]
break
with TemporaryDirectory('_umd2oeb', keep=True) as tmp_dir:
log.debug('Process TOC ...')
oeb.container = DirContainer(tmp_dir, log)
content, cover = book.chapters, book.cover
if content:
for i, ch in enumerate(content):
ch_title, ch_content = ch.title, ch.content
if ch_title is None or ch_content is None:
continue
ch_content = ch_content.replace("\u2029", "")
ch_fn = Path(tmp_dir) / f"ch_{i:04d}.html"
ch_fn.write_text(convert_basic(ch_content, title=ch_title))
oeb.toc.add(ch_title, ch_fn.name)
id_, href = oeb.manifest.generate(id='html',
href=ch_fn.name)
item = oeb.manifest.add(id_, href, 'text/html')
item.html_input_href = ch_fn.name
oeb.spine.add(item, True)
if cover:
cover_file = Path(tmp_dir) / "cover.jpeg"
cover_file.write_bytes(cover)
id_, href = oeb.manifest.generate(id='image',
href=cover_file.name)
oeb.guide.add('cover', 'Cover', href)
return oeb
def test_lopen():
from calibre.ptempfile import TemporaryDirectory
from calibre import CurrentDir
n = u'f\xe4llen'
print('testing lopen()')
if iswindows:
import msvcrt, win32api
def assert_not_inheritable(f):
if win32api.GetHandleInformation(msvcrt.get_osfhandle(f.fileno())) & 0b1:
raise SystemExit('File handle is inheritable!')
else:
def assert_not_inheritable(f):
if not fcntl.fcntl(f, fcntl.F_GETFD) & fcntl.FD_CLOEXEC:
raise SystemExit('File handle is inheritable!')
def copen(*args):
ans = lopen(*args)
assert_not_inheritable(ans)
return ans
with TemporaryDirectory() as tdir, CurrentDir(tdir):
with copen(n, 'w') as f:
f.write('one')
print 'O_CREAT tested'
with copen(n, 'w+b') as f:
f.write('two')
with copen(n, 'r') as f:
if f.read() == 'two':
print 'O_TRUNC tested'
else:
raise Exception('O_TRUNC failed')
with copen(n, 'ab') as f:
f.write('three')
with copen(n, 'r+') as f:
if f.read() == 'twothree':
print 'O_APPEND tested'
else:
raise Exception('O_APPEND failed')
with copen(n, 'r+') as f:
f.seek(3)
f.write('xxxxx')
f.seek(0)
if f.read() == 'twoxxxxx':
print 'O_RANDOM tested'
else:
raise Exception('O_RANDOM failed')
def render_cover(cpage, zf, reader=None):
from calibre.ebooks import render_html_svg_workaround
from calibre.utils.logging import default_log
if not cpage:
return
if reader is not None and reader.encryption_meta.is_encrypted(cpage):
return
with TemporaryDirectory('_epub_meta') as tdir:
with CurrentDir(tdir):
zf.extractall()
cpage = os.path.join(tdir, cpage)
if not os.path.exists(cpage):
return
return render_html_svg_workaround(cpage, default_log)
def test_ssl(self):
'Test serving over SSL'
address = '127.0.0.1'
with TemporaryDirectory('srv-test-ssl') as tdir:
cert_file, key_file, ca_file = map(lambda x:os.path.join(tdir, x), 'cka')
create_server_cert(address, ca_file, cert_file, key_file, key_size=1024)
ctx = ssl.create_default_context(cafile=ca_file)
with TestServer(lambda data:(data.path[0] + data.read()), ssl_certfile=cert_file, ssl_keyfile=key_file, listen_on=address, port=0) as server:
conn = httplib.HTTPSConnection(address, server.address[1], strict=True, context=ctx)
conn.request('GET', '/test', 'body')
r = conn.getresponse()
self.ae(r.status, httplib.OK)
self.ae(r.read(), b'testbody')
cert = conn.sock.getpeercert()
subject = dict(x[0] for x in cert['subject'])
self.ae(subject['commonName'], address)
def serve_forever(self):
""" Listen for incoming connections. """
if self.pre_activated_socket is None:
try:
self.do_bind()
except socket.error as err:
if not self.opts.fallback_to_detected_interface:
raise
ip = get_external_ip()
if ip == self.bind_address[0]:
raise
self.log.warn(
'Failed to bind to %s with error: %s. Trying to bind to the default interface: %s instead'
% (self.bind_address[0], as_unicode(err), ip))
self.bind_address = (ip, self.bind_address[1])
self.do_bind()
else:
self.socket = self.pre_activated_socket
self.pre_activated_socket = None
self.setup_socket()
self.connection_map = {}
self.socket.listen(min(socket.SOMAXCONN, 128))
self.bound_address = ba = self.socket.getsockname()
if isinstance(ba, tuple):
ba = ':'.join(map(type(''), ba))
self.pool.start()
with TemporaryDirectory(prefix='srv-') as tdir:
self.tdir = tdir
self.ready = True
if self.LISTENING_MSG:
self.log(self.LISTENING_MSG, ba)
self.plugin_pool.start()
while self.ready:
try:
self.tick()
except SystemExit:
self.shutdown()
raise
except KeyboardInterrupt:
break
except:
self.log.exception('Error in ServerLoop.tick')
self.shutdown()
def download_external_resources(container, urls, timeout=60, progress_report=lambda url, done, total: None):
failures = {}
replacements = {}
data_uri_map = {}
with TemporaryDirectory('editor-download') as tdir:
pool = Pool(10)
with closing(pool):
for ok, result in pool.imap_unordered(partial(download_one, tdir, timeout, progress_report, data_uri_map), urls):
if ok:
url, suggested_filename, downloaded_file, mt = result
with lopen(downloaded_file, 'rb') as src:
name = container.add_file(suggested_filename, src, mt, modify_name_if_needed=True)
replacements[url] = name
else:
url, err = result
failures[url] = err
return replacements, failures
def test_library_restrictions(self): # {{{
from calibre.srv.opts import Options
from calibre.srv.handler import Handler
from calibre.db.legacy import create_backend
opts = Options(userdb=':memory:')
Data = namedtuple('Data', 'username')
with TemporaryDirectory() as base:
l1, l2, l3 = map(lambda x: os.path.join(base, 'l' + x), '123')
for l in (l1, l2, l3):
create_backend(l).close()
ctx = Handler((l1, l2, l3), opts).router.ctx
um = ctx.user_manager
def get_library(username=None, library_id=None):
ans = ctx.get_library(Data(username), library_id=library_id)
return os.path.basename(ans.backend.library_path)
def library_info(username=None):
lmap, defaultlib = ctx.library_info(Data(username))
lmap = {k: os.path.basename(v) for k, v in lmap.iteritems()}
return lmap, defaultlib
self.assertEqual(get_library(), 'l1')
self.assertEqual(library_info()[0],
{'l%d' % i: 'l%d' % i
for i in range(1, 4)})
self.assertEqual(library_info()[1], 'l1')
self.assertRaises(HTTPForbidden, get_library, 'xxx')
um.add_user('a', 'a')
self.assertEqual(
library_info('a')[0],
{'l%d' % i: 'l%d' % i
for i in range(1, 4)})
um.update_user_restrictions('a', {'blocked_library_names': ['L2']})
self.assertEqual(
library_info('a')[0],
{'l%d' % i: 'l%d' % i
for i in range(1, 4) if i != 2})
um.update_user_restrictions('a', {'allowed_library_names': ['l3']})
self.assertEqual(
library_info('a')[0],
{'l%d' % i: 'l%d' % i
for i in range(1, 4) if i == 3})
self.assertEqual(library_info('a')[1], 'l3')
self.assertRaises(HTTPForbidden, get_library, 'a', 'l1')
self.assertRaises(HTTPForbidden, get_library, 'xxx')
def get_file(self, path, outfile, end_session=True):
'''
Read the file at ``path`` on the device and write it to outfile.
:param outfile: file object like ``sys.stdout`` or the result of an
:func:`open` call.
'''
if self.bambook:
with TemporaryDirectory() as tdir:
if self.bambook.GetFile(path, tdir):
filepath = os.path.join(tdir, path)
f = file(filepath, 'rb')
outfile.write(f.read())
f.close()
else:
print "Unable to get file from Bambook:", path
def convert_text(self, oeb_book):
from calibre.ebooks.pdf.writer import PDFWriter
from calibre.ebooks.metadata.opf2 import OPF
self.log.debug('Serializing oeb input to disk for processing...')
self.get_cover_data()
with TemporaryDirectory('_pdf_out') as oeb_dir:
from calibre.customize.ui import plugin_for_output_format
oeb_output = plugin_for_output_format('oeb')
oeb_output.convert(oeb_book, oeb_dir, self.input_plugin, self.opts,
self.log)
opfpath = glob.glob(os.path.join(oeb_dir, '*.opf'))[0]
opf = OPF(opfpath, os.path.dirname(opfpath))
self.write(PDFWriter, [s.path for s in opf.spine])
def test_library_move(self):
' Test moving of library '
from calibre.ptempfile import TemporaryDirectory
cache = self.init_cache()
self.assertIn('metadata.db', cache.get_top_level_move_items()[0])
all_ids = cache.all_book_ids()
fmt1 = cache.format(1, 'FMT1')
cov = cache.cover(1)
with TemporaryDirectory('moved_lib') as tdir:
cache.move_library_to(tdir)
self.assertIn('moved_lib', cache.backend.library_path)
self.assertIn('moved_lib', cache.backend.dbpath)
self.assertEqual(fmt1, cache.format(1, 'FMT1'))
self.assertEqual(cov, cache.cover(1))
cache.reload_from_db()
self.assertEqual(all_ids, cache.all_book_ids())
cache.backend.close()
def run_fork(self):
with TemporaryDirectory('_single_metadata_download') as tdir:
self.keep_going = True
t = Thread(target=self.monitor_tdir, args=(tdir,))
t.daemon = True
t.start()
try:
res = fork_job('calibre.ebooks.metadata.sources.worker',
'single_covers',
(self.title, self.authors, self.identifiers, self.caches,
tdir),
no_output=True, abort=self.abort)
self.log.append_dump(res['result'])
finally:
self.keep_going = False
t.join()
def book(db, notify_changes, is_remote, args):
data, fname, fmt, add_duplicates, otitle, oauthors, oisbn, otags, oseries, oseries_index, ocover, oidentifiers, olanguages = args
with add_ctx(), TemporaryDirectory(
'add-single') as tdir, run_import_plugins_before_metadata(tdir):
if is_remote:
with lopen(os.path.join(tdir, fname), 'wb') as f:
f.write(data[1])
path = f.name
else:
path = data
path = run_import_plugins([path])[0]
fmt = os.path.splitext(path)[1]
fmt = (fmt[1:] if fmt else None) or 'unknown'
with lopen(path, 'rb') as stream:
mi = get_metadata(stream,
stream_type=fmt,
use_libprs_metadata=True)
if not mi.title:
mi.title = os.path.splitext(os.path.basename(path))[0]
if not mi.authors:
mi.authors = [_('Unknown')]
if oidentifiers:
ids = mi.get_identifiers()
ids.update(oidentifiers)
mi.set_identifiers(ids)
for x in ('title', 'authors', 'isbn', 'tags', 'series', 'languages'):
val = locals()['o' + x]
if val:
setattr(mi, x, val)
if oseries:
mi.series_index = oseries_index
if ocover:
mi.cover = None
mi.cover_data = ocover
ids, duplicates = db.add_books([(mi, {
fmt: path
})],
add_duplicates=add_duplicates,
run_hooks=False)
if is_remote:
notify_changes(books_added(ids))
db.dump_metadata()
return ids, bool(duplicates), mi.title
def extract_member(path, match=re.compile(r'\.(jpg|jpeg|gif|png)\s*$', re.I),
name=None, as_file=False):
if hasattr(path, 'read'):
data = path.read()
f = NamedTemporaryFile(suffix='.rar')
f.write(data)
f.flush()
path = f.name
path = os.path.abspath(path)
if as_file:
path = _extract_member(path, match, name)
return path, open(path, 'rb')
else:
with TemporaryDirectory('_libunrar') as tdir:
with CurrentDir(tdir):
path = _extract_member(path, match, name)
return path, open(path, 'rb').read()
def test():
from calibre.ptempfile import TemporaryDirectory
from calibre import CurrentDir
from glob import glob
with TemporaryDirectory() as tdir, CurrentDir(tdir):
shutil.copyfile(I('devices/kindle.jpg'), 'test.jpg')
ret = optimize_jpeg('test.jpg')
if ret is not None:
raise SystemExit('optimize_jpeg failed: %s' % ret)
ret = encode_jpeg('test.jpg')
if ret is not None:
raise SystemExit('encode_jpeg failed: %s' % ret)
shutil.copyfile(I('lt.png'), 'test.png')
ret = optimize_png('test.png')
if ret is not None:
raise SystemExit('optimize_png failed: %s' % ret)
if glob('*.bak'):
raise SystemExit('Spurious .bak files left behind')
def convert(self, oeb_book, output_path, input_plugin, opts, log):
from calibre.ebooks.pml.pmlml import PMLMLizer
from calibre.utils.zipfile import ZipFile
with TemporaryDirectory('_pmlz_output') as tdir:
pmlmlizer = PMLMLizer(log)
pml = unicode_type(pmlmlizer.extract_content(oeb_book, opts))
with open(os.path.join(tdir, 'index.pml'), 'wb') as out:
out.write(pml.encode(opts.pml_output_encoding, 'replace'))
img_path = os.path.join(tdir, 'index_img')
if not os.path.exists(img_path):
os.makedirs(img_path)
self.write_images(oeb_book.manifest, pmlmlizer.image_hrefs, img_path, opts)
log.debug('Compressing output...')
pmlz = ZipFile(output_path, 'w')
pmlz.add_dir(tdir)
def get_preprocess_html(path_to_ebook, output=None):
from calibre.ebooks.conversion.plumber import set_regex_wizard_callback, Plumber
from calibre.utils.logging import DevNull
from calibre.ptempfile import TemporaryDirectory
raw = {}
set_regex_wizard_callback(raw.__setitem__)
with TemporaryDirectory('_regex_wiz') as tdir:
pl = Plumber(path_to_ebook,
os.path.join(tdir, 'a.epub'),
DevNull(),
for_regex_wizard=True)
pl.run()
items = [raw[item.href] for item in pl.oeb.spine if item.href in raw]
with (sys.stdout if output is None else open(output, 'wb')) as out:
for html in items:
out.write(html.encode('utf-8'))
out.write(b'\n\n' + b'-' * 80 + b'\n\n')
def run(self):
try:
with TemporaryDirectory('_library_restore') as tdir:
self.library_path = tdir
self.scan_library()
if not self.load_preferences():
# Something went wrong with preferences restore. Start over
# with a new database and attempt to rebuild the structure
# from the metadata in the opf
dbpath = os.path.join(self.library_path, 'metadata.db')
if os.path.exists(dbpath):
os.remove(dbpath)
self.create_cc_metadata()
self.restore_books()
if self.successes == 0 and len(self.dirs) > 0:
raise Exception(('Something bad happened'))
self.replace_db()
except:
self.tb = traceback.format_exc()
def load_jxr_data(data):
with TemporaryDirectory() as tdir:
if iswindows and isinstance(tdir, type('')):
tdir = tdir.encode('mbcs')
with lopen(os.path.join(tdir, 'input.jxr'), 'wb') as f:
f.write(data)
cmd = [
get_exe_path('JxrDecApp'), '-i', 'input.jxr', '-o', 'output.tif'
]
creationflags = 0x08 if iswindows else 0
subprocess.Popen(cmd,
cwd=tdir,
stdout=lopen(os.devnull, 'wb'),
stderr=subprocess.STDOUT,
creationflags=creationflags).wait()
i = QImage()
if not i.load(os.path.join(tdir, 'output.tif')):
raise NotImage('Failed to convert JPEG-XR image')
return i
def tweak(ebook_file):
''' Command line interface to the Tweak Book tool '''
fmt = ebook_file.rpartition('.')[-1].lower()
exploder, rebuilder = get_tools(fmt)
if exploder is None:
prints(
'Cannot tweak %s files. Supported formats are: EPUB, HTMLZ, AZW3, MOBI'
% fmt.upper(),
file=sys.stderr)
raise SystemExit(1)
with TemporaryDirectory(
'_tweak_' +
os.path.basename(ebook_file).rpartition('.')[0]) as tdir:
try:
opf = exploder(ebook_file, tdir, question=ask_cli_question)
except WorkerError as e:
prints('Failed to unpack', ebook_file)
prints(e.orig_tb)
raise SystemExit(1)
except Error as e:
prints(as_unicode(e), file=sys.stderr)
raise SystemExit(1)
if opf is None:
# The question was answered with No
return
prints('Book extracted to', tdir)
prints('Make your tweaks and once you are done,', __appname__,
'will rebuild', ebook_file, 'from', tdir)
print()
proceed = ask_cli_question('Rebuild ' + ebook_file + '?')
if proceed:
prints('Rebuilding', ebook_file, 'please wait ...')
try:
rebuilder(tdir, ebook_file)
except WorkerError as e:
prints('Failed to rebuild', ebook_file)
prints(e.orig_tb)
raise SystemExit(1)
prints(ebook_file, 'successfully tweaked')
def get_metadata(stream, extract_cover=True):
""" Return metadata as a L{MetaInfo} object """
mi = MetaInformation(_('Unknown'), [_('Unknown')])
stream.seek(0)
pml = b''
if stream.name.endswith('.pmlz'):
with TemporaryDirectory('_unpmlz') as tdir:
zf = ZipFile(stream)
zf.extractall(tdir)
pmls = glob.glob(os.path.join(tdir, '*.pml'))
for p in pmls:
with open(p, 'r+b') as p_stream:
pml += p_stream.read()
if extract_cover:
mi.cover_data = get_cover(os.path.splitext(os.path.basename(stream.name))[0], tdir, True)
else:
pml = stream.read()
if extract_cover:
mi.cover_data = get_cover(os.path.splitext(os.path.basename(stream.name))[0], os.path.abspath(os.path.dirname(stream.name)))
for comment in re.findall(br'(?ms)\\v.*?\\v', pml):
m = re.search(br'TITLE="(.*?)"', comment)
if m:
mi.title = re.sub('[\x00-\x1f]', '', prepare_string_for_xml(m.group(1).strip().decode('cp1252', 'replace')))
m = re.search(br'AUTHOR="(.*?)"', comment)
if m:
if mi.authors == [_('Unknown')]:
mi.authors = []
mi.authors.append(re.sub('[\x00-\x1f]', '', prepare_string_for_xml(m.group(1).strip().decode('cp1252', 'replace'))))
m = re.search(br'PUBLISHER="(.*?)"', comment)
if m:
mi.publisher = re.sub('[\x00-\x1f]', '', prepare_string_for_xml(m.group(1).strip().decode('cp1252', 'replace')))
m = re.search(br'COPYRIGHT="(.*?)"', comment)
if m:
mi.rights = re.sub('[\x00-\x1f]', '', prepare_string_for_xml(m.group(1).strip().decode('cp1252', 'replace')))
m = re.search(br'ISBN="(.*?)"', comment)
if m:
mi.isbn = re.sub('[\x00-\x1f]', '', prepare_string_for_xml(m.group(1).strip().decode('cp1252', 'replace')))
return mi
def convert(self, oeb, output_path, input_plugin, opts, log):
self.log, self.opts, self.oeb = log, opts, oeb
lrf_opts = LRFOptions(output_path, opts, oeb)
if input_plugin.is_image_collection:
self.convert_images(input_plugin.get_images(), lrf_opts,
getattr(opts, 'wide', False))
return
self.flatten_toc()
from calibre.ptempfile import TemporaryDirectory
with TemporaryDirectory('_lrf_output') as tdir:
from calibre.customize.ui import plugin_for_output_format
oeb_output = plugin_for_output_format('oeb')
oeb_output.convert(oeb, tdir, input_plugin, opts, log)
opf = [x for x in os.listdir(tdir) if x.endswith('.opf')][0]
from calibre.ebooks.lrf.html.convert_from import process_file
process_file(os.path.join(tdir, opf), lrf_opts, self.log)
def format_group(db, notify_changes, is_remote, args):
formats, add_duplicates = args
with add_ctx(), TemporaryDirectory('add-multiple') as tdir, run_import_plugins_before_metadata(tdir):
if is_remote:
paths = []
for name, data in formats:
with lopen(os.path.join(tdir, os.path.basename(name)), 'wb') as f:
f.write(data)
paths.append(f.name)
else:
paths = list(formats)
paths = run_import_plugins(paths)
mi = metadata_from_formats(paths)
if mi.title is None:
return None, set(), False
ids, dups = db.add_books([(mi, create_format_map(paths))], add_duplicates=add_duplicates, run_hooks=False)
if is_remote:
notify_changes(books_added(ids))
db.dump_metadata()
return mi.title, ids, bool(dups)
def __enter__(self, *args):
if self.plugin_path is not None:
from calibre.utils.zipfile import ZipFile
zf = ZipFile(self.plugin_path)
extensions = set([x.rpartition('.')[-1].lower() for x in
zf.namelist()])
zip_safe = True
for ext in ('pyd', 'so', 'dll', 'dylib'):
if ext in extensions:
zip_safe = False
if zip_safe:
sys.path.insert(0, self.plugin_path)
self.sys_insertion_path = self.plugin_path
else:
from calibre.ptempfile import TemporaryDirectory
self._sys_insertion_tdir = TemporaryDirectory('plugin_unzip')
self.sys_insertion_path = self._sys_insertion_tdir.__enter__(*args)
zf.extractall(self.sys_insertion_path)
sys.path.insert(0, self.sys_insertion_path)
zf.close()
def set_metadata(stream, mi):
with TemporaryDirectory(u'_podofo_set_metadata') as tdir:
with open(os.path.join(tdir, u'input.pdf'), 'wb') as f:
shutil.copyfileobj(stream, f)
try:
touched = fork_job(
'calibre.utils.podofo', 'set_metadata_',
(tdir, mi.title, mi.authors, mi.book_producer, mi.tags))
except WorkerError as e:
raise Exception('Failed to set PDF metadata: %s' % e.orig_tb)
if touched:
with open(os.path.join(tdir, u'output.pdf'), 'rb') as f:
f.seek(0, 2)
if f.tell() > 100:
f.seek(0)
stream.seek(0)
stream.truncate()
shutil.copyfileobj(f, stream)
stream.flush()
stream.seek(0)
def convert_text(self, oeb_book):
from calibre.ebooks.metadata.opf2 import OPF
from calibre.ebooks.pdf.render.from_html import PDFWriter
self.log.debug('Serializing oeb input to disk for processing...')
self.get_cover_data()
self.process_fonts()
if self.opts.pdf_use_document_margins and self.stored_page_margins:
import json
for href, margins in iteritems(self.stored_page_margins):
item = oeb_book.manifest.hrefs.get(href)
if item is not None:
root = item.data
if hasattr(root, 'xpath') and margins:
root.set('data-calibre-pdf-output-page-margins',
json.dumps(margins))
# Remove javascript
for item in self.oeb.spine:
root = item.data
if hasattr(root, 'xpath'):
for script in root.xpath('//*[local-name()="script"]'):
script.text = None
script.attrib.clear()
for elem in root.iter('*'):
for attr in tuple(elem.attrib):
if attr.startswith('on'):
elem.set(attr, '')
with TemporaryDirectory('_pdf_out') as oeb_dir:
from calibre.customize.ui import plugin_for_output_format
oeb_output = plugin_for_output_format('oeb')
oeb_output.convert(oeb_book, oeb_dir, self.input_plugin, self.opts,
self.log)
opfpath = glob.glob(os.path.join(oeb_dir, '*.opf'))[0]
opf = OPF(opfpath, os.path.dirname(opfpath))
self.write(PDFWriter, [s.path for s in opf.spine],
getattr(opf, 'toc', None))
def get_simple_book(fmt='epub'):
cache = get_cache()
ans = os.path.join(cache, 'simple.'+fmt)
src = os.path.join(os.path.dirname(__file__), 'simple.html')
if needs_recompile(ans, src):
with TemporaryDirectory('bpt') as tdir:
with CurrentDir(tdir):
raw = open(src, 'rb').read().decode('utf-8')
raw = add_resources(raw, {
'LMONOI': P('fonts/liberation/LiberationMono-Italic.ttf'),
'LMONOR': P('fonts/liberation/LiberationMono-Regular.ttf'),
'IMAGE1': I('marked.png'),
'IMAGE2': I('textures/light_wood.png'),
})
shutil.copy2(I('lt.png'), '.')
x = 'index.html'
with open(x, 'wb') as f:
f.write(raw.encode('utf-8'))
build_book(x, ans, args=[
'--level1-toc=//h:h2', '--language=en', '--authors=Kovid Goyal', '--cover=lt.png'])
return ans
def do_test(stream):
c = comment(stream)
if c != b'some comment\n':
raise ValueError('Comment not read: %r != %r' % (c, b'some comment\n'))
if set(names(stream)) != {
'1/sub-one', 'one.txt', '2/sub-two.txt', '诶比屁.txt', 'Füße.txt',
'uncompressed', 'max-compressed'}:
raise ValueError('Name list does not match')
with TemporaryDirectory('test-unrar') as tdir:
extract(stream, tdir)
for name in tdata:
if name not in '1 2 symlink'.split():
with open(os.path.join(tdir, name), 'rb') as s:
if s.read() != tdata[name]:
raise ValueError('Did not extract %s properly' % name)
for name in tdata:
if name not in '1 2 symlink'.split():
d = extract_member(stream, name=name)
if d is None or d[1] != tdata[name]:
raise ValueError(
'Failed to extract %s %r != %r' % (name, d, tdata[name]))
def convert(self, oeb_book, output_path, input_plugin, opts, log):
from calibre.ebooks.oeb.base import OEB_IMAGES, xml2str
from calibre.utils.zipfile import ZipFile
with TemporaryDirectory('_txtz_output') as tdir:
# TXT
txt_name = 'index.txt'
if opts.txt_output_formatting.lower() == 'textile':
txt_name = 'index.text'
with TemporaryFile(txt_name) as tf:
TXTOutput.convert(self, oeb_book, tf, input_plugin, opts, log)
shutil.copy(tf, os.path.join(tdir, txt_name))
# Images
for item in oeb_book.manifest:
if item.media_type in OEB_IMAGES:
if hasattr(self.writer, 'images'):
path = os.path.join(tdir, 'images')
if item.href in self.writer.images:
href = self.writer.images[item.href]
else:
continue
else:
path = os.path.join(tdir, os.path.dirname(item.href))
href = os.path.basename(item.href)
if not os.path.exists(path):
os.makedirs(path)
with open(os.path.join(path, href), 'wb') as imgf:
imgf.write(item.data)
# Metadata
with open(os.path.join(tdir, 'metadata.opf'), 'wb') as mdataf:
root = oeb_book.metadata.to_opf1()
elem = root.makeelement('text-formatting')
elem.text = opts.txt_output_formatting
root.append(elem)
mdataf.write(xml2str(root, pretty_print=True))
txtz = ZipFile(output_path, 'w')
txtz.add_dir(tdir)
def __enter__(self, processed=False, only_input_plugin=False,
run_char_count=True, read_anchor_map=True, view_kepub=False, read_links=True):
''' Convert an ebook file into an exploded OEB book suitable for
display in viewers/preprocessing etc. '''
from calibre.ebooks.conversion.plumber import Plumber, create_oebbook
self.delete_on_exit = []
self._tdir = TemporaryDirectory('_ebook_iter')
self.base = self._tdir.__enter__()
plumber = Plumber(self.pathtoebook, self.base, self.log, view_kepub=view_kepub)
plumber.setup_options()
if self.pathtoebook.lower().endswith('.opf'):
plumber.opts.dont_package = True
if hasattr(plumber.opts, 'no_process'):
plumber.opts.no_process = True
plumber.input_plugin.for_viewer = True
with plumber.input_plugin, open(plumber.input, 'rb') as inf:
self.pathtoopf = plumber.input_plugin(inf,
plumber.opts, plumber.input_fmt, self.log,
{}, self.base)
if not only_input_plugin:
# Run the HTML preprocess/parsing from the conversion pipeline as
# well
if (processed or plumber.input_fmt.lower() in {'pdb', 'pdf', 'rb'}
and not hasattr(self.pathtoopf, 'manifest')):
if hasattr(self.pathtoopf, 'manifest'):
self.pathtoopf = write_oebbook(self.pathtoopf, self.base)
self.pathtoopf = create_oebbook(self.log, self.pathtoopf,
plumber.opts)
if hasattr(self.pathtoopf, 'manifest'):
self.pathtoopf = write_oebbook(self.pathtoopf, self.base)
self.book_format = os.path.splitext(self.pathtoebook)[1][1:].upper()
if getattr(plumber.input_plugin, 'is_kf8', False):
fs = ':joint' if getattr(plumber.input_plugin, 'mobi_is_joint', False) else ''
self.book_format = 'KF8' + fs
self.opf = getattr(plumber.input_plugin, 'optimize_opf_parsing', None)
if self.opf is None:
self.opf = OPF(self.pathtoopf, os.path.dirname(self.pathtoopf))
self.language = self.opf.language
if self.language:
self.language = self.language.lower()
ordered = [i for i in self.opf.spine if i.is_linear] + \
[i for i in self.opf.spine if not i.is_linear]
self.spine = []
Spiny = partial(SpineItem, read_anchor_map=read_anchor_map, read_links=read_links,
run_char_count=run_char_count, from_epub=self.book_format == 'EPUB')
is_comic = plumber.input_fmt.lower() in {'cbc', 'cbz', 'cbr', 'cb7'}
for i in ordered:
spath = i.path
mt = None
if i.idref is not None:
mt = self.opf.manifest.type_for_id(i.idref)
if mt is None:
mt = guess_type(spath)[0]
try:
self.spine.append(Spiny(spath, mime_type=mt))
if is_comic:
self.spine[-1].is_single_page = True
except:
self.log.warn('Missing spine item:', repr(spath))
cover = self.opf.cover
if cover and self.ebook_ext in {'lit', 'mobi', 'prc', 'opf', 'fb2',
'azw', 'azw3', 'docx', 'htmlz'}:
cfile = os.path.join(self.base, 'calibre_iterator_cover.html')
rcpath = os.path.relpath(cover, self.base).replace(os.sep, '/')
chtml = (TITLEPAGE%prepare_string_for_xml(rcpath, True)).encode('utf-8')
with open(cfile, 'wb') as f:
f.write(chtml)
self.spine[0:0] = [Spiny(cfile,
mime_type='application/xhtml+xml')]
self.delete_on_exit.append(cfile)
if self.opf.path_to_html_toc is not None and \
self.opf.path_to_html_toc not in self.spine:
try:
self.spine.append(Spiny(self.opf.path_to_html_toc))
except:
import traceback
traceback.print_exc()
sizes = [i.character_count for i in self.spine]
self.pages = [math.ceil(i/float(self.CHARACTERS_PER_PAGE)) for i in sizes]
for p, s in zip(self.pages, self.spine):
s.pages = p
start = 1
for s in self.spine:
s.start_page = start
start += s.pages
s.max_page = s.start_page + s.pages - 1
self.toc = self.opf.toc
if read_anchor_map:
create_indexing_data(self.spine, self.toc)
self.verify_links()
self.read_bookmarks()
return self
class Plugin(object): # {{{
'''
A calibre plugin. Useful members include:
* ``self.plugin_path``: Stores path to the zip file that contains
this plugin or None if it is a builtin
plugin
* ``self.site_customization``: Stores a customization string entered
by the user.
Methods that should be overridden in sub classes:
* :meth:`initialize`
* :meth:`customization_help`
Useful methods:
* :meth:`temporary_file`
* :meth:`__enter__`
* :meth:`load_resources`
'''
#: List of platforms this plugin works on.
#: For example: ``['windows', 'osx', 'linux']``
supported_platforms = []
#: The name of this plugin. You must set it something other
#: than Trivial Plugin for it to work.
name = 'Trivial Plugin'
#: The version of this plugin as a 3-tuple (major, minor, revision)
version = (1, 0, 0)
#: A short string describing what this plugin does
description = _('Does absolutely nothing')
#: The author of this plugin
author = _('Unknown')
#: When more than one plugin exists for a filetype,
#: the plugins are run in order of decreasing priority
#: i.e. plugins with higher priority will be run first.
#: The highest possible priority is ``sys.maxint``.
#: Default priority is 1.
priority = 1
#: The earliest version of calibre this plugin requires
minimum_calibre_version = (0, 4, 118)
#: If False, the user will not be able to disable this plugin. Use with
#: care.
can_be_disabled = True
#: The type of this plugin. Used for categorizing plugins in the
#: GUI
type = _('Base')
def __init__(self, plugin_path):
self.plugin_path = plugin_path
self.site_customization = None
def initialize(self):
'''
Called once when calibre plugins are initialized. Plugins are
re-initialized every time a new plugin is added. Also note that if the
plugin is run in a worker process, such as for adding books, then the
plugin will be initialized for every new worker process.
Perform any plugin specific initialization here, such as extracting
resources from the plugin zip file. The path to the zip file is
available as ``self.plugin_path``.
Note that ``self.site_customization`` is **not** available at this point.
'''
pass
def config_widget(self):
'''
Implement this method and :meth:`save_settings` in your plugin to
use a custom configuration dialog, rather then relying on the simple
string based default customization.
This method, if implemented, must return a QWidget. The widget can have
an optional method validate() that takes no arguments and is called
immediately after the user clicks OK. Changes are applied if and only
if the method returns True.
If for some reason you cannot perform the configuration at this time,
return a tuple of two strings (message, details), these will be
displayed as a warning dialog to the user and the process will be
aborted.
'''
raise NotImplementedError()
def save_settings(self, config_widget):
'''
Save the settings specified by the user with config_widget.
:param config_widget: The widget returned by :meth:`config_widget`.
'''
raise NotImplementedError()
def do_user_config(self, parent=None):
'''
This method shows a configuration dialog for this plugin. It returns
True if the user clicks OK, False otherwise. The changes are
automatically applied.
'''
from PyQt5.Qt import QDialog, QDialogButtonBox, QVBoxLayout, \
QLabel, Qt, QLineEdit
from calibre.gui2 import gprefs
prefname = 'plugin config dialog:'+self.type + ':' + self.name
geom = gprefs.get(prefname, None)
config_dialog = QDialog(parent)
button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
v = QVBoxLayout(config_dialog)
def size_dialog():
if geom is None:
config_dialog.resize(config_dialog.sizeHint())
else:
config_dialog.restoreGeometry(geom)
button_box.accepted.connect(config_dialog.accept)
button_box.rejected.connect(config_dialog.reject)
config_dialog.setWindowTitle(_('Customize') + ' ' + self.name)
try:
config_widget = self.config_widget()
except NotImplementedError:
config_widget = None
if isinstance(config_widget, tuple):
from calibre.gui2 import warning_dialog
warning_dialog(parent, _('Cannot configure'), config_widget[0],
det_msg=config_widget[1], show=True)
return False
if config_widget is not None:
v.addWidget(config_widget)
v.addWidget(button_box)
size_dialog()
config_dialog.exec_()
if config_dialog.result() == QDialog.Accepted:
if hasattr(config_widget, 'validate'):
if config_widget.validate():
self.save_settings(config_widget)
else:
self.save_settings(config_widget)
else:
from calibre.customize.ui import plugin_customization, \
customize_plugin
help_text = self.customization_help(gui=True)
help_text = QLabel(help_text, config_dialog)
help_text.setWordWrap(True)
help_text.setTextInteractionFlags(Qt.LinksAccessibleByMouse | Qt.LinksAccessibleByKeyboard)
help_text.setOpenExternalLinks(True)
v.addWidget(help_text)
sc = plugin_customization(self)
if not sc:
sc = ''
sc = sc.strip()
sc = QLineEdit(sc, config_dialog)
v.addWidget(sc)
v.addWidget(button_box)
size_dialog()
config_dialog.exec_()
if config_dialog.result() == QDialog.Accepted:
sc = unicode(sc.text()).strip()
customize_plugin(self, sc)
geom = bytearray(config_dialog.saveGeometry())
gprefs[prefname] = geom
return config_dialog.result()
def load_resources(self, names):
'''
If this plugin comes in a ZIP file (user added plugin), this method
will allow you to load resources from the ZIP file.
For example to load an image::
pixmap = QPixmap()
pixmap.loadFromData(self.load_resources(['images/icon.png']).itervalues().next())
icon = QIcon(pixmap)
:param names: List of paths to resources in the zip file using / as separator
:return: A dictionary of the form ``{name: file_contents}``. Any names
that were not found in the zip file will not be present in the
dictionary.
'''
if self.plugin_path is None:
raise ValueError('This plugin was not loaded from a ZIP file')
ans = {}
with zipfile.ZipFile(self.plugin_path, 'r') as zf:
for candidate in zf.namelist():
if candidate in names:
ans[candidate] = zf.read(candidate)
return ans
def customization_help(self, gui=False):
'''
Return a string giving help on how to customize this plugin.
By default raise a :class:`NotImplementedError`, which indicates that
the plugin does not require customization.
If you re-implement this method in your subclass, the user will
be asked to enter a string as customization for this plugin.
The customization string will be available as
``self.site_customization``.
Site customization could be anything, for example, the path to
a needed binary on the user's computer.
:param gui: If True return HTML help, otherwise return plain text help.
'''
raise NotImplementedError
def temporary_file(self, suffix):
'''
Return a file-like object that is a temporary file on the file system.
This file will remain available even after being closed and will only
be removed on interpreter shutdown. Use the ``name`` member of the
returned object to access the full path to the created temporary file.
:param suffix: The suffix that the temporary file will have.
'''
return PersistentTemporaryFile(suffix)
def is_customizable(self):
try:
self.customization_help()
return True
except NotImplementedError:
return False
def __enter__(self, *args):
'''
Add this plugin to the python path so that it's contents become directly importable.
Useful when bundling large python libraries into the plugin. Use it like this::
with plugin:
import something
'''
if self.plugin_path is not None:
from calibre.utils.zipfile import ZipFile
zf = ZipFile(self.plugin_path)
extensions = set([x.rpartition('.')[-1].lower() for x in
zf.namelist()])
zip_safe = True
for ext in ('pyd', 'so', 'dll', 'dylib'):
if ext in extensions:
zip_safe = False
break
if zip_safe:
sys.path.insert(0, self.plugin_path)
self.sys_insertion_path = self.plugin_path
else:
from calibre.ptempfile import TemporaryDirectory
self._sys_insertion_tdir = TemporaryDirectory('plugin_unzip')
self.sys_insertion_path = self._sys_insertion_tdir.__enter__(*args)
zf.extractall(self.sys_insertion_path)
sys.path.insert(0, self.sys_insertion_path)
zf.close()
def __exit__(self, *args):
ip, it = getattr(self, 'sys_insertion_path', None), getattr(self,
'_sys_insertion_tdir', None)
if ip in sys.path:
sys.path.remove(ip)
if hasattr(it, '__exit__'):
it.__exit__(*args)
def cli_main(self, args):
'''
This method is the main entry point for your plugins command line
interface. It is called when the user does: calibre-debug -r "Plugin
Name". Any arguments passed are present in the args variable.
'''
raise NotImplementedError('The %s plugin has no command line interface'
%self.name)
def __enter__(self, processed=False, only_input_plugin=False,
run_char_count=True, read_anchor_map=True, view_kepub=False, read_links=True):
''' Convert an ebook file into an exploded OEB book suitable for
display in viewers/preprocessing etc. '''
self.delete_on_exit = []
self._tdir = TemporaryDirectory('_ebook_iter')
self.base = self._tdir.__enter__()
self.book_format, self.pathtoopf, input_fmt = run_extract_book(
self.pathtoebook, self.base, only_input_plugin=only_input_plugin, view_kepub=view_kepub, processed=processed)
self.opf = OPF(self.pathtoopf, os.path.dirname(self.pathtoopf))
self.language = self.opf.language
if self.language:
self.language = self.language.lower()
ordered = [i for i in self.opf.spine if i.is_linear] + \
[i for i in self.opf.spine if not i.is_linear]
self.spine = []
Spiny = partial(SpineItem, read_anchor_map=read_anchor_map, read_links=read_links,
run_char_count=run_char_count, from_epub=self.book_format == 'EPUB')
is_comic = input_fmt.lower() in {'cbc', 'cbz', 'cbr', 'cb7'}
for i in ordered:
spath = i.path
mt = None
if i.idref is not None:
mt = self.opf.manifest.type_for_id(i.idref)
if mt is None:
mt = guess_type(spath)[0]
try:
self.spine.append(Spiny(spath, mime_type=mt))
if is_comic:
self.spine[-1].is_single_page = True
except:
self.log.warn('Missing spine item:', repr(spath))
cover = self.opf.cover
if cover and self.ebook_ext in {'lit', 'mobi', 'prc', 'opf', 'fb2',
'azw', 'azw3', 'docx', 'htmlz'}:
cfile = os.path.join(self.base, 'calibre_iterator_cover.html')
rcpath = os.path.relpath(cover, self.base).replace(os.sep, '/')
chtml = (TITLEPAGE%prepare_string_for_xml(rcpath, True)).encode('utf-8')
with open(cfile, 'wb') as f:
f.write(chtml)
self.spine[0:0] = [Spiny(cfile,
mime_type='application/xhtml+xml')]
self.delete_on_exit.append(cfile)
if self.opf.path_to_html_toc is not None and \
self.opf.path_to_html_toc not in self.spine:
try:
self.spine.append(Spiny(self.opf.path_to_html_toc))
except:
import traceback
traceback.print_exc()
sizes = [i.character_count for i in self.spine]
self.pages = [math.ceil(i/float(self.CHARACTERS_PER_PAGE)) for i in sizes]
for p, s in zip(self.pages, self.spine):
s.pages = p
start = 1
for s in self.spine:
s.start_page = start
start += s.pages
s.max_page = s.start_page + s.pages - 1
self.toc = self.opf.toc
if read_anchor_map:
create_indexing_data(self.spine, self.toc)
self.verify_links()
self.read_bookmarks()
return self
class EbookIterator(BookmarksMixin):
CHARACTERS_PER_PAGE = 1000
def __init__(self, pathtoebook, log=None, copy_bookmarks_to_file=True):
BookmarksMixin.__init__(self, copy_bookmarks_to_file=copy_bookmarks_to_file)
self.log = log or default_log
pathtoebook = pathtoebook.strip()
self.pathtoebook = os.path.abspath(pathtoebook)
self.config = DynamicConfig(name='iterator')
ext = os.path.splitext(pathtoebook)[1].replace('.', '').lower()
ext = re.sub(r'(x{0,1})htm(l{0,1})', 'html', ext)
self.ebook_ext = ext.replace('original_', '')
def search(self, text, index, backwards=False):
from calibre.ebooks.oeb.polish.parsing import parse
pmap = [(i, path) for i, path in enumerate(self.spine)]
if backwards:
pmap.reverse()
q = text.lower()
for i, path in pmap:
if (backwards and i < index) or (not backwards and i > index):
with open(path, 'rb') as f:
raw = f.read().decode(path.encoding)
root = parse(raw)
fragments = []
def serialize(elem):
if elem.text:
fragments.append(elem.text.lower())
if elem.tail:
fragments.append(elem.tail.lower())
for child in elem.iterchildren():
if hasattr(getattr(child, 'tag', None), 'rpartition') and child.tag.rpartition('}')[-1] not in {'script', 'style', 'del'}:
serialize(child)
elif getattr(child, 'tail', None):
fragments.append(child.tail.lower())
for body in root.xpath('//*[local-name() = "body"]'):
body.tail = None
serialize(body)
if q in ''.join(fragments):
return i
def __enter__(self, processed=False, only_input_plugin=False,
run_char_count=True, read_anchor_map=True, view_kepub=False, read_links=True):
''' Convert an ebook file into an exploded OEB book suitable for
display in viewers/preprocessing etc. '''
self.delete_on_exit = []
self._tdir = TemporaryDirectory('_ebook_iter')
self.base = self._tdir.__enter__()
self.book_format, self.pathtoopf, input_fmt = run_extract_book(
self.pathtoebook, self.base, only_input_plugin=only_input_plugin, view_kepub=view_kepub, processed=processed)
self.opf = OPF(self.pathtoopf, os.path.dirname(self.pathtoopf))
self.language = self.opf.language
if self.language:
self.language = self.language.lower()
ordered = [i for i in self.opf.spine if i.is_linear] + \
[i for i in self.opf.spine if not i.is_linear]
self.spine = []
Spiny = partial(SpineItem, read_anchor_map=read_anchor_map, read_links=read_links,
run_char_count=run_char_count, from_epub=self.book_format == 'EPUB')
is_comic = input_fmt.lower() in {'cbc', 'cbz', 'cbr', 'cb7'}
for i in ordered:
spath = i.path
mt = None
if i.idref is not None:
mt = self.opf.manifest.type_for_id(i.idref)
if mt is None:
mt = guess_type(spath)[0]
try:
self.spine.append(Spiny(spath, mime_type=mt))
if is_comic:
self.spine[-1].is_single_page = True
except:
self.log.warn('Missing spine item:', repr(spath))
cover = self.opf.cover
if cover and self.ebook_ext in {'lit', 'mobi', 'prc', 'opf', 'fb2',
'azw', 'azw3', 'docx', 'htmlz'}:
cfile = os.path.join(self.base, 'calibre_iterator_cover.html')
rcpath = os.path.relpath(cover, self.base).replace(os.sep, '/')
chtml = (TITLEPAGE%prepare_string_for_xml(rcpath, True)).encode('utf-8')
with open(cfile, 'wb') as f:
f.write(chtml)
self.spine[0:0] = [Spiny(cfile,
mime_type='application/xhtml+xml')]
self.delete_on_exit.append(cfile)
if self.opf.path_to_html_toc is not None and \
self.opf.path_to_html_toc not in self.spine:
try:
self.spine.append(Spiny(self.opf.path_to_html_toc))
except:
import traceback
traceback.print_exc()
sizes = [i.character_count for i in self.spine]
self.pages = [math.ceil(i/float(self.CHARACTERS_PER_PAGE)) for i in sizes]
for p, s in zip(self.pages, self.spine):
s.pages = p
start = 1
for s in self.spine:
s.start_page = start
start += s.pages
s.max_page = s.start_page + s.pages - 1
self.toc = self.opf.toc
if read_anchor_map:
create_indexing_data(self.spine, self.toc)
self.verify_links()
self.read_bookmarks()
return self
def verify_links(self):
spine_paths = {s:s for s in self.spine}
for item in self.spine:
base = os.path.dirname(item)
for link in item.all_links:
try:
p = urlparse(urlunquote(link))
except Exception:
continue
if not p.scheme and not p.netloc:
path = os.path.abspath(os.path.join(base, p.path)) if p.path else item
try:
path = spine_paths[path]
except Exception:
continue
if not p.fragment or p.fragment in path.anchor_map:
item.verified_links.add((path, p.fragment))
def __exit__(self, *args):
self._tdir.__exit__(*args)
for x in self.delete_on_exit:
try:
os.remove(x)
except:
pass
class EbookIterator(BookmarksMixin):
CHARACTERS_PER_PAGE = 1000
def __init__(self, pathtoebook, log=None):
self.log = log or default_log
pathtoebook = pathtoebook.strip()
self.pathtoebook = os.path.abspath(pathtoebook)
self.config = DynamicConfig(name='iterator')
ext = os.path.splitext(pathtoebook)[1].replace('.', '').lower()
ext = re.sub(r'(x{0,1})htm(l{0,1})', 'html', ext)
self.ebook_ext = ext.replace('original_', '')
def search(self, text, index, backwards=False):
text = prepare_string_for_xml(text.lower())
pmap = [(i, path) for i, path in enumerate(self.spine)]
if backwards:
pmap.reverse()
for i, path in pmap:
if (backwards and i < index) or (not backwards and i > index):
with open(path, 'rb') as f:
raw = f.read().decode(path.encoding)
try:
raw = xml_replace_entities(raw)
except:
pass
if text in raw.lower():
return i
def __enter__(self, processed=False, only_input_plugin=False,
run_char_count=True, read_anchor_map=True, view_kepub=False, read_links=True):
''' Convert an ebook file into an exploded OEB book suitable for
display in viewers/preprocessing etc. '''
from calibre.ebooks.conversion.plumber import Plumber, create_oebbook
self.delete_on_exit = []
self._tdir = TemporaryDirectory('_ebook_iter')
self.base = self._tdir.__enter__()
plumber = Plumber(self.pathtoebook, self.base, self.log, view_kepub=view_kepub)
plumber.setup_options()
if self.pathtoebook.lower().endswith('.opf'):
plumber.opts.dont_package = True
if hasattr(plumber.opts, 'no_process'):
plumber.opts.no_process = True
plumber.input_plugin.for_viewer = True
with plumber.input_plugin, open(plumber.input, 'rb') as inf:
self.pathtoopf = plumber.input_plugin(inf,
plumber.opts, plumber.input_fmt, self.log,
{}, self.base)
if not only_input_plugin:
# Run the HTML preprocess/parsing from the conversion pipeline as
# well
if (processed or plumber.input_fmt.lower() in {'pdb', 'pdf', 'rb'}
and not hasattr(self.pathtoopf, 'manifest')):
if hasattr(self.pathtoopf, 'manifest'):
self.pathtoopf = write_oebbook(self.pathtoopf, self.base)
self.pathtoopf = create_oebbook(self.log, self.pathtoopf,
plumber.opts)
if hasattr(self.pathtoopf, 'manifest'):
self.pathtoopf = write_oebbook(self.pathtoopf, self.base)
self.book_format = os.path.splitext(self.pathtoebook)[1][1:].upper()
if getattr(plumber.input_plugin, 'is_kf8', False):
fs = ':joint' if getattr(plumber.input_plugin, 'mobi_is_joint', False) else ''
self.book_format = 'KF8' + fs
self.opf = getattr(plumber.input_plugin, 'optimize_opf_parsing', None)
if self.opf is None:
self.opf = OPF(self.pathtoopf, os.path.dirname(self.pathtoopf))
self.language = self.opf.language
if self.language:
self.language = self.language.lower()
ordered = [i for i in self.opf.spine if i.is_linear] + \
[i for i in self.opf.spine if not i.is_linear]
self.spine = []
Spiny = partial(SpineItem, read_anchor_map=read_anchor_map, read_links=read_links,
run_char_count=run_char_count, from_epub=self.book_format == 'EPUB')
is_comic = plumber.input_fmt.lower() in {'cbc', 'cbz', 'cbr', 'cb7'}
for i in ordered:
spath = i.path
mt = None
if i.idref is not None:
mt = self.opf.manifest.type_for_id(i.idref)
if mt is None:
mt = guess_type(spath)[0]
try:
self.spine.append(Spiny(spath, mime_type=mt))
if is_comic:
self.spine[-1].is_single_page = True
except:
self.log.warn('Missing spine item:', repr(spath))
cover = self.opf.cover
if cover and self.ebook_ext in {'lit', 'mobi', 'prc', 'opf', 'fb2',
'azw', 'azw3', 'docx', 'htmlz'}:
cfile = os.path.join(self.base, 'calibre_iterator_cover.html')
rcpath = os.path.relpath(cover, self.base).replace(os.sep, '/')
chtml = (TITLEPAGE%prepare_string_for_xml(rcpath, True)).encode('utf-8')
with open(cfile, 'wb') as f:
f.write(chtml)
self.spine[0:0] = [Spiny(cfile,
mime_type='application/xhtml+xml')]
self.delete_on_exit.append(cfile)
if self.opf.path_to_html_toc is not None and \
self.opf.path_to_html_toc not in self.spine:
try:
self.spine.append(Spiny(self.opf.path_to_html_toc))
except:
import traceback
traceback.print_exc()
sizes = [i.character_count for i in self.spine]
self.pages = [math.ceil(i/float(self.CHARACTERS_PER_PAGE)) for i in sizes]
for p, s in zip(self.pages, self.spine):
s.pages = p
start = 1
for s in self.spine:
s.start_page = start
start += s.pages
s.max_page = s.start_page + s.pages - 1
self.toc = self.opf.toc
if read_anchor_map:
create_indexing_data(self.spine, self.toc)
self.verify_links()
self.read_bookmarks()
return self
def verify_links(self):
spine_paths = {s:s for s in self.spine}
for item in self.spine:
base = os.path.dirname(item)
for link in item.all_links:
try:
p = urlparse(urlunquote(link))
except Exception:
continue
if not p.scheme and not p.netloc:
path = os.path.abspath(os.path.join(base, p.path)) if p.path else item
try:
path = spine_paths[path]
except Exception:
continue
if not p.fragment or p.fragment in path.anchor_map:
item.verified_links.add((path, p.fragment))
def __exit__(self, *args):
self._tdir.__exit__(*args)
for x in self.delete_on_exit:
try:
os.remove(x)
except:
pass
