def __init__(self, current_cover=None, parent=None):
QDialog.__init__(self, parent)
self.current_cover = current_cover
self.log = Log()
self.book = self.cover_pixmap = None
self.setWindowTitle(_('Downloading metadata...'))
self.setWindowIcon(QIcon(I('download-metadata.png')))
self.stack = QStackedWidget()
self.l = l = QVBoxLayout()
self.setLayout(l)
l.addWidget(self.stack)
self.bb = QDialogButtonBox(QDialogButtonBox.Cancel
| QDialogButtonBox.Ok)
self.h = h = QHBoxLayout()
l.addLayout(h)
self.bb.rejected.connect(self.reject)
self.bb.accepted.connect(self.accept)
self.ok_button = self.bb.button(self.bb.Ok)
self.ok_button.setEnabled(False)
self.ok_button.clicked.connect(self.ok_clicked)
self.prev_button = pb = QPushButton(QIcon(I('back.png')), _('&Back'),
self)
pb.clicked.connect(self.back_clicked)
pb.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)
self.log_button = self.bb.addButton(_('&View log'), self.bb.ActionRole)
self.log_button.clicked.connect(self.view_log)
self.log_button.setIcon(QIcon(I('debug.png')))
self.prev_button.setVisible(False)
h.addWidget(self.prev_button), h.addWidget(self.bb)
self.identify_widget = IdentifyWidget(self.log, self)
self.identify_widget.rejected.connect(self.reject)
self.identify_widget.results_found.connect(self.identify_results_found)
self.identify_widget.book_selected.connect(self.book_selected)
self.stack.addWidget(self.identify_widget)
self.covers_widget = CoversWidget(self.log,
self.current_cover,
parent=self)
self.covers_widget.chosen.connect(self.ok_clicked)
self.stack.addWidget(self.covers_widget)
self.resize(850, 600)
geom = gprefs.get('metadata_single_gui_geom', None)
if geom is not None and geom:
self.restoreGeometry(geom)
self.finished.connect(self.cleanup)
def setup_pipeline(self, *args):
oidx = self.groups.currentIndex().row()
output_format = self.output_format
input_path = 'dummy.epub'
output_path = 'dummy.' + output_format
log = Log()
log.outputs = []
self.plumber = Plumber(input_path,
output_path,
log,
merge_plugin_recs=False)
self.plumber.merge_plugin_recs(self.plumber.output_plugin)
def widget_factory(cls):
return cls(self, self.plumber.get_option_by_name,
self.plumber.get_option_help, self.db)
self.setWindowTitle(
ngettext(_('Bulk convert one book'), _('Bulk convert {} books'),
self.num_of_books).format(self.num_of_books))
lf = widget_factory(LookAndFeelWidget)
hw = widget_factory(HeuristicsWidget)
sr = widget_factory(SearchAndReplaceWidget)
ps = widget_factory(PageSetupWidget)
sd = widget_factory(StructureDetectionWidget)
toc = widget_factory(TOCWidget)
toc.manually_fine_tune_toc.hide()
output_widget = self.plumber.output_plugin.gui_configuration_widget(
self, self.plumber.get_option_by_name,
self.plumber.get_option_help, self.db)
self.break_cycles()
widgets = self.widgets = [lf, hw, ps, sd, toc, sr]
if output_widget is not None:
widgets.append(output_widget)
for w in widgets:
w.set_help_signal.connect(self.help.setPlainText)
w.setVisible(False)
self._groups_model = GroupModel(widgets)
self.groups.setModel(self._groups_model)
idx = oidx if -1 < oidx < self._groups_model.rowCount() else 0
self.groups.setCurrentIndex(self._groups_model.index(idx))
self.show_pane(idx)
try:
shutil.rmtree(self.plumber.archive_input_tdir, ignore_errors=True)
except:
pass
def gui_catalog(fmt,
title,
dbspec,
ids,
out_file_name,
sync,
fmt_options,
connected_device,
notification=DummyReporter(),
log=None):
if log is None:
log = Log()
from calibre.library import db
from calibre.utils.config import prefs
prefs.refresh()
db = db(read_only=True)
db.catalog_plugin_on_device_temp_mapping = dbspec
# Create a minimal OptionParser that we can append to
parser = OptionParser()
args = []
parser.add_option("--verbose",
action="store_true",
dest="verbose",
default=True)
opts, args = parser.parse_args()
# Populate opts
# opts.gui_search_text = something
opts.catalog_title = title
opts.connected_device = connected_device
opts.ids = ids
opts.search_text = None
opts.sort_by = None
opts.sync = sync
# Extract the option dictionary to comma-separated lists
for option in fmt_options:
if isinstance(fmt_options[option], list):
setattr(opts, option, ','.join(fmt_options[option]))
else:
setattr(opts, option, fmt_options[option])
# Fetch and run the plugin for fmt
# Returns 0 if successful, 1 if no catalog built
plugin = plugin_for_catalog_format(fmt)
return plugin.run(out_file_name, opts, db, notification=notification)
def main(args=sys.argv):
log = Log()
parser, plumber = create_option_parser(args, log)
opts, leftover_args = parser.parse_args(args)
if len(leftover_args) > 3:
log.error('Extra arguments not understood:',
u', '.join(leftover_args[3:]))
return 1
for x in ('read_metadata_from_opf', 'cover'):
if getattr(opts, x, None) is not None:
setattr(opts, x, abspath(getattr(opts, x)))
if opts.search_replace:
opts.search_replace = read_sr_patterns(opts.search_replace, log)
if opts.transform_css_rules:
from calibre.ebooks.css_transform_rules import import_rules, validate_rule
with open(opts.transform_css_rules, 'rb') as tcr:
opts.transform_css_rules = rules = list(import_rules(tcr.read()))
for rule in rules:
title, msg = validate_rule(rule)
if title and msg:
log.error('Failed to parse CSS transform rules')
log.error(title)
log.error(msg)
return 1
recommendations = [(n.dest, getattr(opts,
n.dest), OptionRecommendation.HIGH)
for n in parser.options_iter() if n.dest]
plumber.merge_ui_recommendations(recommendations)
try:
plumber.run()
except ConversionUserFeedBack as e:
ll = {
'info': log.info,
'warn': log.warn,
'error': log.error
}.get(e.level, log.info)
ll(e.title)
if e.det_msg:
log.debug(e.detmsg)
ll(e.msg)
raise SystemExit(1)
log(_('Output saved to'), ' ', plumber.output)
return 0
def genesis(self, gui):
log = Log()
log.outputs = []
self.plumber = Plumber('dummy.epub',
'dummy.epub',
log,
dummy=True,
merge_plugin_recs=False)
def widget_factory(cls):
plugin = getattr(cls, 'conv_plugin', None)
if plugin is None:
hfunc = self.plumber.get_option_help
else:
options = plugin.options.union(plugin.common_options)
def hfunc(name):
for rec in options:
if rec.option == name:
ans = getattr(rec, 'help', None)
if ans is not None:
return ans.replace(
'%default',
unicode_type(rec.recommended_value))
return cls(self, self.plumber.get_option_by_name, hfunc, None,
None)
self.load_conversion_widgets()
widgets = list(map(widget_factory, self.conversion_widgets))
self.model = Model(widgets)
self.list.setModel(self.model)
for w in widgets:
w.changed_signal.connect(self.changed_signal)
sa = QScrollArea(self)
sa.setWidget(w)
sa.setWidgetResizable(True)
self.stack.addWidget(sa)
if isinstance(w, TOCWidget):
w.manually_fine_tune_toc.hide()
self.list.current_changed.connect(self.category_current_changed)
self.list.setCurrentIndex(self.model.index(0))
def convert_document(source: Path, target: Path, cover_dir: Path):
post = frontmatter.load(source)
log = Log()
title = post.get("title", "Untitled")
author = post.get("creator", "Shannan Lekwati")
args = [
("authors", author),
("language", post.get("lang", "en")),
("title", title),
]
date = post.get("date")
if date:
args += [
("pubdate", str(post["date"])),
("timestamp", str(post["date"]))
]
summary = post.get("summary")
if summary:
args += [ ("comments", summary) ]
cover_image = post.get("cover", {}).get("image")
if cover_image:
cover_path = cover_dir.joinpath(cover_image).absolute()
if cover_path.exists():
args += [ ("cover", str(cover_path)) ]
else:
print(f"WARNING: {cover_image} in {source} does not exists", file=sys.stderr)
with NamedTemporaryFile(suffix=source.suffix, mode="w") as f:
f.write(f"# {title} by **{author}** \n")
if "dedication" in post:
f.write(f"{dedication}\n")
f.write(post.content)
f.flush()
plumber = Plumber(f.name, target, log)
recommendations = [(k, v, OptionRecommendation.HIGH) for (k,v) in args]
plumber.merge_ui_recommendations(recommendations)
plumber.run()
def cli_main(self, argv):
self.cli = True
log = JobLog(Log())
self.report_version(log)
log.info("")
allowed_exts = [".epub", ".opf", ".mobi", ".doc", ".docx", ".kpf", ".kfx-zip"]
ext_choices = ", ".join(allowed_exts[:-1] + ["or " + allowed_exts[-1]])
parser = argparse.ArgumentParser(prog='calibre-debug -r "KFX Output" --', description="Convert e-book to KFX format")
parser.add_argument("-a", "--asin", action="store", help="Optional ASIN to assign to the book")
parser.add_argument("-c", "--clean", action="store_true", help="Save the input file cleaned for conversion to KFX")
parser.add_argument("-d", "--doc", action="store_true", help="Create personal document (PDOC) instead of book (EBOK)")
parser.add_argument("-p", "--pages", action="store", type=int, default=-1,
help="Create n approximate page numbers if missing from input file (0 for auto)")
parser.add_argument("-t", "--timeout", action="store_true", help="Stop conversions lasting over 15 minutes")
parser.add_argument("-l", "--logs", action="store_true", help="Show log files produced during conversion")
parser.add_argument("infile", help="Pathname of the %s file to be converted" % ext_choices)
parser.add_argument("outfile", nargs="?", help="Optional pathname of the resulting .kfx file")
args = parser.parse_args(argv[1:])
input = book_name = args.infile
intype = os.path.splitext(input)[1]
if not os.path.isfile(input):
raise Exception("Input file does not exist: %s" % input)
if args.outfile:
output = args.outfile
else:
output = os.path.join(os.path.dirname(input), os.path.splitext(os.path.basename(input))[0] + ".kfx")
if not output.endswith(".kfx"):
raise Exception("Output file must have .kfx extension")
if intype in [".kpf", ".kfx-zip"]:
self.convert_from_kpf_or_zip(log, book_name, input, args.asin, args.doc, args.pages, intype == ".kpf", output)
elif intype in allowed_exts:
self.convert_using_previewer(
log, book_name, input, args.asin, args.doc, args.pages, args.logs,
args.clean, TIMEOUT if args.timeout else None, output)
else:
raise Exception("Input file must be %s" % ext_choices)
def gui_polish(data):
files = data.pop('files')
if not data.pop('metadata'):
data.pop('opf')
if not data.pop('do_cover'):
data.pop('cover', None)
file_map = {x:x for x in files}
opts = ALL_OPTS.copy()
opts.update(data)
O = namedtuple('Options', ' '.join(ALL_OPTS.iterkeys()))
opts = O(**opts)
log = Log(level=Log.DEBUG)
report = []
polish(file_map, opts, log, report.append)
log('')
log(REPORT)
for msg in report:
log(msg)
return '\n\n'.join(report)
def gui_convert(input,
output,
recommendations,
notification=DummyReporter(),
abort_after_input_dump=False,
log=None,
override_input_metadata=False):
recommendations = list(recommendations)
recommendations.append(('verbose', 2, OptionRecommendation.HIGH))
if log is None:
log = Log()
plumber = Plumber(input,
output,
log,
report_progress=notification,
abort_after_input_dump=abort_after_input_dump,
override_input_metadata=override_input_metadata)
plumber.merge_ui_recommendations(recommendations)
plumber.run()
def main(args=None):
parser = option_parser()
opts, args = parser.parse_args(args or sys.argv[1:])
log = Log(level=Log.DEBUG if opts.verbose else Log.INFO)
if not args:
parser.print_help()
log.error(_('You must provide the input file to polish'))
raise SystemExit(1)
if len(args) > 2:
parser.print_help()
log.error(_('Unknown extra arguments'))
raise SystemExit(1)
if len(args) == 1:
inbook = args[0]
base, ext = inbook.rpartition('.')[0::2]
outbook = base + '_polished.' + ext
else:
inbook, outbook = args
popts = ALL_OPTS.copy()
for k, v in popts.items():
popts[k] = getattr(opts, k, None)
O = namedtuple('Options', ' '.join(iter(popts.keys())))
popts = O(**popts)
report = []
if not tuple(
[_f for _f in (getattr(popts, name) for name in ALL_OPTS) if _f]):
parser.print_help()
log.error(_('You must specify at least one action to perform'))
raise SystemExit(1)
polish({inbook: outbook}, popts, log, report.append)
log('')
log(REPORT)
for msg in report:
log(msg)
log('Output written to:', outbook)
def main(args=sys.argv):
log = Log()
parser = option_parser()
if len(args) < 2:
print 'Error: No command sepecified.\n'
print_help(parser, log)
return 1
command = args[1].lower().strip()
if command in COMMANDS.keys():
del args[1]
return COMMANDS[command].main(args, command)
else:
parser.parse_args(args)
print 'Unknown command %s.\n' % command
print_help(parser, log)
return 1
# We should never get here.
return 0
def convert_book(path_to_ebook, opf_path, cover_path, output_fmt, recs):
from calibre.customize.conversion import OptionRecommendation
from calibre.ebooks.conversion.plumber import Plumber
from calibre.utils.logging import Log
recs.append(('verbose', 2, OptionRecommendation.HIGH))
recs.append(('read_metadata_from_opf', opf_path,
OptionRecommendation.HIGH))
if cover_path:
recs.append(('cover', cover_path, OptionRecommendation.HIGH))
log = Log()
os.chdir(os.path.dirname(path_to_ebook))
status_file = share_open('status', 'wb')
def notification(percent, msg=''):
status_file.write('{}:{}|||\n'.format(percent, msg).encode('utf-8'))
status_file.flush()
output_path = os.path.abspath('output.' + output_fmt.lower())
plumber = Plumber(path_to_ebook, output_path, log,
report_progress=notification, override_input_metadata=True)
plumber.merge_ui_recommendations(recs)
plumber.run()
def process_epub(input_file, output_file):
from calibre import CurrentDir
from calibre.libunzip import extract as zipextract
from calibre.ptempfile import TemporaryDirectory
from calibre.utils.logging import Log
from calibre_plugins.modify_epub.container import ExtendedContainer
input_file = os.path.abspath(input_file)
output_file = os.path.abspath(output_file)
# Extract the epub into a temp directory
with TemporaryDirectory('fb2-hyphens') as tdir:
with CurrentDir(tdir):
zipextract(input_file, tdir)
# Use our own simplified wrapper around an ePub that will
# preserve the file structure and css
container = ExtendedContainer(tdir, Log())
is_modified = process_epub_file(container)
if is_modified:
container.write(output_file)
def main(args=sys.argv):
log = Log()
parser, plumber = create_option_parser(args, log)
opts, leftover_args = parser.parse_args(args)
if len(leftover_args) > 3:
log.error('Extra arguments not understood:',
u', '.join(leftover_args[3:]))
return 1
for x in ('read_metadata_from_opf', 'cover'):
if getattr(opts, x, None) is not None:
setattr(opts, x, abspath(getattr(opts, x)))
if opts.search_replace:
opts.search_replace = read_sr_patterns(opts.search_replace, log)
recommendations = [(n.dest, getattr(opts, n.dest),
OptionRecommendation.HIGH) \
for n in parser.options_iter()
if n.dest]
plumber.merge_ui_recommendations(recommendations)
try:
plumber.run()
except ConversionUserFeedBack as e:
ll = {
'info': log.info,
'warn': log.warn,
'error': log.error
}.get(e.level, log.info)
ll(e.title)
if e.det_msg:
log.debug(e.detmsg)
ll(e.msg)
raise SystemExit(1)
log(_('Output saved to'), ' ', plumber.output)
return 0
def genesis(self, gui):
log = Log()
log.outputs = []
self.plumber = Plumber('dummy.epub', 'dummy.epub', log, dummy=True,
merge_plugin_recs=False)
def widget_factory(cls):
return cls(self, self.plumber.get_option_by_name,
self.plumber.get_option_help, None, None)
self.load_conversion_widgets()
widgets = list(map(widget_factory, self.conversion_widgets))
self.model = Model(widgets)
self.list.setModel(self.model)
for w in widgets:
w.changed_signal.connect(self.changed_signal)
self.stack.addWidget(w)
if isinstance(w, TOCWidget):
w.manually_fine_tune_toc.hide()
self.list.currentChanged = self.category_current_changed
self.list.setCurrentIndex(self.model.index(0))
import json
from calibre.utils.logging import Log
from calibre.ebooks.metadata.book.base import Metadata
from calibre_plugins.crossref_doi_download import DoiMeta
from calibre_plugins.crossref_doi_download.doi_reader import DoiReader, get_title, get_author_list
from calibre_plugins.crossref_doi_download.doi_request import DoiQuery
from polyglot.urllib import urlencode
import calibre_plugins.crossref_doi_download.short_doi as sd
log = Log()
reader = DoiReader(log)
dm = DoiMeta('./plugin/')
br = dm.browser
onlineQuery = DoiQuery(br, log)
short_doi = '10/aabbe'
short_doi = '10.1002/9781118895047'
# cdata = br.open(doiquery).read()
# json.loads(cdata)
sd.is_short_doi(short_doi)
sd.return_full_doi(br, short_doi)
# https://doi.org/api/handles/10/aabbe
# doilink = 'https://dx.doi.org/10.1002/(SICI)1097-0258(19980815/30)17:15/16%3C1661::AID-SIM968%3E3.0.CO;2-2?noredirect'
# fullurl = 'https://api.crossref.org/works?%s' % dois
# fullurl = 'https://api.crossref.org/works?filter=doi:10/aabbe&mailto=vikoya5988%40oniaj.com'
# cdata = br.open(fullurl).read()
# data = json.loads(cdata)
# message = data['message']
def create_fetcher(options, image_map={}, log=None):
if log is None:
log = Log(level=Log.DEBUG) if options.verbose else Log()
return RecursiveFetcher(options, log, image_map={})
def run(self, opts):
from calibre.utils.serialize import msgpack_dumps
scripts = {}
for x in ('console', 'gui'):
for name in basenames[x]:
if name in ('calibre-complete', 'calibre_postinstall'):
continue
scripts[name] = x
dest = self.j(self.RESOURCES, 'scripts.calibre_msgpack')
if self.newer(dest, self.j(self.SRC, 'calibre', 'linux.py')):
self.info('\tCreating ' + self.b(dest))
with open(dest, 'wb') as f:
f.write(msgpack_dumps(scripts))
from calibre.web.feeds.recipes.collection import \
serialize_builtin_recipes, iterate_over_builtin_recipe_files
files = [x[1] for x in iterate_over_builtin_recipe_files()]
dest = self.j(self.RESOURCES, 'builtin_recipes.xml')
if self.newer(dest, files):
self.info('\tCreating builtin_recipes.xml')
xml = serialize_builtin_recipes()
with open(dest, 'wb') as f:
f.write(xml)
recipe_icon_dir = self.a(
self.j(self.RESOURCES, '..', 'recipes', 'icons'))
dest = os.path.splitext(dest)[0] + '.zip'
files += glob.glob(self.j(recipe_icon_dir, '*.png'))
if self.newer(dest, files):
self.info('\tCreating builtin_recipes.zip')
with zipfile.ZipFile(dest, 'w', zipfile.ZIP_STORED) as zf:
for n in sorted(files, key=self.b):
with open(n, 'rb') as f:
zf.writestr(self.b(n), f.read())
dest = self.j(self.RESOURCES, 'ebook-convert-complete.calibre_msgpack')
files = []
for x in os.walk(self.j(self.SRC, 'calibre')):
for f in x[-1]:
if f.endswith('.py'):
files.append(self.j(x[0], f))
if self.newer(dest, files):
self.info('\tCreating ' + self.b(dest))
complete = {}
from calibre.ebooks.conversion.plumber import supported_input_formats
complete['input_fmts'] = set(supported_input_formats())
from calibre.web.feeds.recipes.collection import get_builtin_recipe_titles
complete['input_recipes'] = [
t + '.recipe ' for t in get_builtin_recipe_titles()
]
from calibre.customize.ui import available_output_formats
complete['output'] = set(available_output_formats())
from calibre.ebooks.conversion.cli import create_option_parser
from calibre.utils.logging import Log
log = Log()
# log.outputs = []
for inf in supported_input_formats():
if inf in ('zip', 'rar', 'oebzip'):
continue
for ouf in available_output_formats():
of = ouf if ouf == 'oeb' else 'dummy.' + ouf
p = create_option_parser(('ec', 'dummy1.' + inf, of, '-h'),
log)[0]
complete[(inf, ouf)] = [
x + ' ' for x in get_opts_from_parser(p)
]
with open(dest, 'wb') as f:
f.write(msgpack_dumps(only_unicode_recursive(complete)))
self.info('\tCreating template-functions.json')
dest = self.j(self.RESOURCES, 'template-functions.json')
function_dict = {}
import inspect
from calibre.utils.formatter_functions import formatter_functions
for obj in formatter_functions().get_builtins().values():
eval_func = inspect.getmembers(
obj,
lambda x: inspect.ismethod(x) and x.__name__ == 'evaluate')
try:
lines = [
l[4:] for l in inspect.getsourcelines(eval_func[0][1])[0]
]
except:
continue
lines = ''.join(lines)
function_dict[obj.name] = lines
dump_json(function_dict, dest)
self.info('\tCreating editor-functions.json')
dest = self.j(self.RESOURCES, 'editor-functions.json')
function_dict = {}
from calibre.gui2.tweak_book.function_replace import builtin_functions
for func in builtin_functions():
try:
src = ''.join(inspect.getsourcelines(func)[0][1:])
except Exception:
continue
src = src.replace('def ' + func.__name__, 'def replace')
imports = [
f'from {x.__module__} import {x.__name__}'
for x in func.imports
]
if imports:
src = '\n'.join(imports) + '\n\n' + src
function_dict[func.name] = src
dump_json(function_dict, dest)
self.info('\tCreating user-manual-translation-stats.json')
d = {}
for lc, stats in iteritems(
json.load(
open(
self.j(self.d(self.SRC), 'manual', 'locale',
'completed.json')))):
total = sum(itervalues(stats))
d[lc] = stats['translated'] / float(total)
dump_json(d,
self.j(self.RESOURCES, 'user-manual-translation-stats.json'))
src = self.j(self.SRC, '..', 'Changelog.txt')
dest = self.j(self.RESOURCES, 'changelog.json')
if self.newer(dest, [src]):
self.info('\tCreating changelog.json')
from setup.changelog import parse
with open(src, encoding='utf-8') as f:
dump_json(parse(f.read(), parse_dates=False), dest)
def do_download_for_worker(book, options, merge, notification=lambda x, y: x):
'''
Child job, to download story when run as a worker job
'''
from calibre_plugins.fanficfare_plugin import FanFicFareBase
fffbase = FanFicFareBase(options['plugin_path'])
with fffbase: # so the sys.path was modified while loading the
# plug impl.
from calibre_plugins.fanficfare_plugin.dialogs import (
NotGoingToDownload, OVERWRITE, OVERWRITEALWAYS, UPDATE,
UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY, CALIBREONLYSAVECOL)
from calibre_plugins.fanficfare_plugin.fanficfare import adapters, writers, exceptions
from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import get_update_data
from calibre_plugins.fanficfare_plugin.fff_util import (
get_fff_adapter, get_fff_config)
try:
## No need to download at all. Can happen now due to
## collision moving into book for CALIBREONLY changing to
## ADDNEW when story URL not in library.
if book['collision'] in (CALIBREONLY, CALIBREONLYSAVECOL):
logger.info("Skipping CALIBREONLY 'update' down inside worker")
return book
book['comment'] = _('Download started...')
configuration = get_fff_config(book['url'], options['fileform'],
options['personal.ini'])
if configuration.getConfig('use_ssl_unverified_context'):
## monkey patch to avoid SSL bug. dupliated from
## fff_plugin.py because bg jobs run in own process
## space.
import ssl
if hasattr(ssl, '_create_unverified_context'):
ssl._create_default_https_context = ssl._create_unverified_context
if not options[
'updateepubcover'] and 'epub_for_update' in book and book[
'collision'] in (UPDATE, UPDATEALWAYS):
configuration.set("overrides", "never_make_cover", "true")
# images only for epub, html, even if the user mistakenly
# turned it on else where.
if options['fileform'] not in ("epub", "html"):
configuration.set("overrides", "include_images", "false")
adapter = adapters.getAdapter(configuration, book['url'])
adapter.is_adult = book['is_adult']
adapter.username = book['username']
adapter.password = book['password']
adapter.setChaptersRange(book['begin'], book['end'])
configuration.load_cookiejar(options['cookiejarfile'])
#logger.debug("cookiejar:%s"%configuration.cookiejar)
configuration.set_pagecache(options['pagecache'])
story = adapter.getStoryMetadataOnly()
if not story.getMetadata("series") and 'calibre_series' in book:
adapter.setSeries(book['calibre_series'][0],
book['calibre_series'][1])
# set PI version instead of default.
if 'version' in options:
story.setMetadata('version', options['version'])
book['title'] = story.getMetadata("title", removeallentities=True)
book['author_sort'] = book['author'] = story.getList(
"author", removeallentities=True)
book['publisher'] = story.getMetadata("publisher")
book['url'] = story.getMetadata("storyUrl", removeallentities=True)
book['tags'] = story.getSubjectTags(removeallentities=True)
book['comments'] = story.get_sanitized_description()
book['series'] = story.getMetadata("series",
removeallentities=True)
if story.getMetadataRaw('datePublished'):
book['pubdate'] = story.getMetadataRaw(
'datePublished').replace(tzinfo=local_tz)
if story.getMetadataRaw('dateUpdated'):
book['updatedate'] = story.getMetadataRaw(
'dateUpdated').replace(tzinfo=local_tz)
if story.getMetadataRaw('dateCreated'):
book['timestamp'] = story.getMetadataRaw(
'dateCreated').replace(tzinfo=local_tz)
else:
book['timestamp'] = datetime.now().replace(
tzinfo=local_tz) # need *something* there for calibre.
writer = writers.getWriter(options['fileform'], configuration,
adapter)
outfile = book['outfile']
## checks were done earlier, it's new or not dup or newer--just write it.
if book['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
('epub_for_update' not in book and book['collision'] in (UPDATE, UPDATEALWAYS)):
# preserve logfile even on overwrite.
if 'epub_for_update' in book:
adapter.logfile = get_update_data(
book['epub_for_update'])[6]
# change the existing entries id to notid so
# write_epub writes a whole new set to indicate overwrite.
if adapter.logfile:
adapter.logfile = adapter.logfile.replace(
"span id", "span notid")
if book['collision'] == OVERWRITE and 'fileupdated' in book:
lastupdated = story.getMetadataRaw('dateUpdated')
fileupdated = book['fileupdated']
# updated doesn't have time (or is midnight), use dates only.
# updated does have time, use full timestamps.
if (lastupdated.time() == time.min and fileupdated.date() > lastupdated.date()) or \
(lastupdated.time() != time.min and fileupdated > lastupdated):
raise NotGoingToDownload(
_("Not Overwriting, web site is not newer."),
'edit-undo.png',
showerror=False)
logger.info("write to %s" % outfile)
inject_cal_cols(book, story, configuration)
writer.writeStory(outfilename=outfile, forceOverwrite=True)
book['comment'] = _('Download %s completed, %s chapters.') % (
options['fileform'], story.getMetadata("numChapters"))
book['all_metadata'] = story.getAllMetadata(
removeallentities=True)
if options['savemetacol'] != '':
book['savemetacol'] = story.dump_html_metadata()
## checks were done earlier, just update it.
elif 'epub_for_update' in book and book['collision'] in (
UPDATE, UPDATEALWAYS):
# update now handled by pre-populating the old images and
# chapters in the adapter rather than merging epubs.
#urlchaptercount = int(story.getMetadata('numChapters').replace(',',''))
# returns int adjusted for start-end range.
urlchaptercount = story.getChapterCount()
(url, chaptercount, adapter.oldchapters, adapter.oldimgs,
adapter.oldcover, adapter.calibrebookmark, adapter.logfile,
adapter.oldchaptersmap,
adapter.oldchaptersdata) = get_update_data(
book['epub_for_update'])[0:9]
# dup handling from fff_plugin needed for anthology updates.
if book['collision'] == UPDATE:
if chaptercount == urlchaptercount:
if merge:
book['comment'] = _(
"Already contains %d chapters. Reuse as is."
) % chaptercount
book['all_metadata'] = story.getAllMetadata(
removeallentities=True)
if options['savemetacol'] != '':
book['savemetacol'] = story.dump_html_metadata(
)
book['outfile'] = book[
'epub_for_update'] # for anthology merge ops.
return book
else: # not merge,
raise NotGoingToDownload(
_("Already contains %d chapters.") %
chaptercount,
'edit-undo.png',
showerror=False)
elif chaptercount > urlchaptercount:
raise NotGoingToDownload(
_("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update."
) % (chaptercount, urlchaptercount),
'dialog_error.png')
elif chaptercount == 0:
raise NotGoingToDownload(
_("FanFicFare doesn't recognize chapters in existing epub, epub is probably from a different source. Use Overwrite to force update."
), 'dialog_error.png')
if not (book['collision'] == UPDATEALWAYS and chaptercount == urlchaptercount) \
and adapter.getConfig("do_update_hook"):
chaptercount = adapter.hookForUpdates(chaptercount)
logger.info("Do update - epub(%d) vs url(%d)" %
(chaptercount, urlchaptercount))
logger.info("write to %s" % outfile)
inject_cal_cols(book, story, configuration)
writer.writeStory(outfilename=outfile, forceOverwrite=True)
book['comment'] = _('Update %s completed, added %s chapters for %s total.')%\
(options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
book['all_metadata'] = story.getAllMetadata(
removeallentities=True)
if options['savemetacol'] != '':
book['savemetacol'] = story.dump_html_metadata()
else:
## Shouldn't ever get here, but hey, it happened once
## before with prefs['collision']
raise Exception(
"Impossible state reached -- Book: %s:\nOptions:%s:" %
(book, options))
if options['do_wordcount'] == SAVE_YES or (
options['do_wordcount'] == SAVE_YES_UNLESS_SITE
and not story.getMetadataRaw('numWords')):
wordcount = get_word_count(outfile)
logger.info("get_word_count:%s" % wordcount)
story.setMetadata('numWords', wordcount)
writer.writeStory(outfilename=outfile, forceOverwrite=True)
book['all_metadata'] = story.getAllMetadata(
removeallentities=True)
if options['savemetacol'] != '':
book['savemetacol'] = story.dump_html_metadata()
if options['smarten_punctuation'] and options['fileform'] == "epub" \
and calibre_version >= (0, 9, 39):
# for smarten punc
from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
from calibre.utils.logging import Log
from collections import namedtuple
# do smarten_punctuation from calibre's polish feature
data = {'smarten_punctuation': True}
opts = ALL_OPTS.copy()
opts.update(data)
O = namedtuple('Options', ' '.join(six.iterkeys(ALL_OPTS)))
opts = O(**opts)
log = Log(level=Log.DEBUG)
polish({outfile: outfile}, opts, log, logger.info)
except NotGoingToDownload as d:
book['good'] = False
book['status'] = _('Bad')
book['showerror'] = d.showerror
book['comment'] = unicode(d)
book['icon'] = d.icon
except Exception as e:
book['good'] = False
book['status'] = _('Error')
book['comment'] = unicode(e)
book['icon'] = 'dialog_error.png'
book['status'] = _('Error')
logger.info("Exception: %s:%s" % (book, book['comment']),
exc_info=True)
#time.sleep(10)
return book
def set_metadata(self, stream, mi, type_):
from calibre_plugins.kfx_output.kfxlib import (set_logger, YJ_Book, YJ_Metadata)
from calibre.ebooks import normalize as normalize_unicode
from calibre.ebooks.metadata import author_to_author_sort
from calibre.utils.config_base import tweaks
from calibre.utils.date import (is_date_undefined, isoformat)
from calibre.utils.logging import Log
from calibre.utils.localization import (canonicalize_lang, lang_as_iso639_1)
def mapped_author_to_author_sort(author):
if hasattr(mi, "author_sort_map"):
author_sort = mi.author_sort_map.get(author) # use mapping if provided
if author_sort:
return author_sort
return author_to_author_sort(author)
def normalize(s):
if not isinstance(s, type("")):
s = s.decode("utf8", "ignore")
return normalize_unicode(s)
log = set_logger(Log())
filename = stream.name if hasattr(stream, "name") else "stream"
log.info("KFX metadata writer activated for %s" % filename)
try:
from calibre.ebooks.conversion.config import load_defaults
prefs = load_defaults('kfx_output')
except Exception:
prefs = {}
log.info("Failed to read default KFX Output preferences")
md = YJ_Metadata(author_sort_fn=mapped_author_to_author_sort)
md.title = normalize(mi.title)
md.authors = [normalize(author) for author in mi.authors]
if mi.publisher:
md.publisher = normalize(mi.publisher)
if mi.pubdate and not is_date_undefined(mi.pubdate):
md.issue_date = str(isoformat(mi.pubdate)[:10])
if mi.comments:
# Strip user annotations
a_offset = mi.comments.find('<div class="user_annotations">')
ad_offset = mi.comments.find('<hr class="annotations_divider" />')
if a_offset >= 0:
mi.comments = mi.comments[:a_offset]
if ad_offset >= 0:
mi.comments = mi.comments[:ad_offset]
md.description = normalize(mi.comments)
if not mi.is_null('language'):
lang = canonicalize_lang(mi.language)
lang = lang_as_iso639_1(lang) or lang
if lang:
md.language = normalize(lang)
if mi.cover_data[1]:
md.cover_image_data = mi.cover_data
elif mi.cover:
md.cover_image_data = ("jpg", open(mi.cover, 'rb').read())
if not tweaks.get("kfx_output_ignore_asin_metadata", False):
value = mi.identifiers.get("mobi-asin")
if value is not None and re.match(ASIN_RE, value):
md.asin = value
else:
for ident, value in mi.identifiers.items():
if ident.startswith("amazon") and re.match(ASIN_RE, value):
md.asin = value
break
else:
value = mi.identifiers.get("asin")
if value is not None and re.match(ASIN_RE, value):
md.asin = value
if md.asin:
md.cde_content_type = "EBOK"
if prefs.get("approximate_pages", False):
page_count = 0
number_of_pages_field = prefs.get("number_of_pages_field", AUTO_PAGES)
if number_of_pages_field and number_of_pages_field != AUTO_PAGES:
number_of_pages = mi.get(number_of_pages_field, "")
try:
page_count = int(number_of_pages)
except Exception:
pass
else:
page_count = -1
book = YJ_Book(stream, log)
book.decode_book(set_metadata=md, set_approximate_pages=page_count)
new_data = book.convert_to_single_kfx()
set_logger()
stream.seek(0)
stream.truncate()
stream.write(new_data)
stream.seek(0)
def setup_pipeline(self, *args):
oidx = self.groups.currentIndex().row()
input_format = self.input_format
output_format = self.output_format
output_path = 'dummy.' + output_format
log = Log()
log.outputs = []
input_file = 'dummy.' + input_format
if input_format in ARCHIVE_FMTS:
input_file = 'dummy.html'
self.plumber = Plumber(input_file, output_path, log)
def widget_factory(cls):
return cls(self.stack, self.plumber.get_option_by_name,
self.plumber.get_option_help, self.db, self.book_id)
self.mw = widget_factory(MetadataWidget)
self.setWindowTitle(_('Convert') + ' ' + unicode(self.mw.title.text()))
lf = widget_factory(LookAndFeelWidget)
hw = widget_factory(HeuristicsWidget)
sr = widget_factory(SearchAndReplaceWidget)
ps = widget_factory(PageSetupWidget)
sd = widget_factory(StructureDetectionWidget)
toc = widget_factory(TOCWidget)
debug = widget_factory(DebugWidget)
output_widget = None
name = self.plumber.output_plugin.name.lower().replace(' ', '_')
try:
output_widget = importlib.import_module('calibre.gui2.convert.' +
name)
pw = output_widget.PluginWidget
pw.ICON = I('back.png')
pw.HELP = _('Options specific to the output format.')
output_widget = widget_factory(pw)
except ImportError:
pass
input_widget = None
name = self.plumber.input_plugin.name.lower().replace(' ', '_')
try:
input_widget = importlib.import_module('calibre.gui2.convert.' +
name)
pw = input_widget.PluginWidget
pw.ICON = I('forward.png')
pw.HELP = _('Options specific to the input format.')
input_widget = widget_factory(pw)
except ImportError:
pass
while True:
c = self.stack.currentWidget()
if not c: break
self.stack.removeWidget(c)
widgets = [self.mw, lf, hw, ps, sd, toc, sr]
if input_widget is not None:
widgets.append(input_widget)
if output_widget is not None:
widgets.append(output_widget)
widgets.append(debug)
for w in widgets:
self.stack.addWidget(w)
self.connect(w, SIGNAL('set_help(PyQt_PyObject)'),
self.help.setPlainText)
self._groups_model = GroupModel(widgets)
self.groups.setModel(self._groups_model)
idx = oidx if -1 < oidx < self._groups_model.rowCount() else 0
self.groups.setCurrentIndex(self._groups_model.index(idx))
self.stack.setCurrentIndex(idx)
try:
shutil.rmtree(self.plumber.archive_input_tdir, ignore_errors=True)
except:
pass
def run(self, opts):
scripts = {}
for x in ('console', 'gui'):
for name in basenames[x]:
if name in ('calibre-complete', 'calibre_postinstall'):
continue
scripts[name] = x
dest = self.j(self.RESOURCES, 'scripts.pickle')
if self.newer(dest, self.j(self.SRC, 'calibre', 'linux.py')):
self.info('\tCreating scripts.pickle')
f = open(dest, 'wb')
cPickle.dump(scripts, f, -1)
from calibre.web.feeds.recipes.collection import \
serialize_builtin_recipes, iterate_over_builtin_recipe_files
files = [x[1] for x in iterate_over_builtin_recipe_files()]
dest = self.j(self.RESOURCES, 'builtin_recipes.xml')
if self.newer(dest, files):
self.info('\tCreating builtin_recipes.xml')
xml = serialize_builtin_recipes()
with open(dest, 'wb') as f:
f.write(xml)
recipe_icon_dir = self.a(
self.j(self.RESOURCES, '..', 'recipes', 'icons'))
dest = os.path.splitext(dest)[0] + '.zip'
files += glob.glob(self.j(recipe_icon_dir, '*.png'))
if self.newer(dest, files):
self.info('\tCreating builtin_recipes.zip')
with zipfile.ZipFile(dest, 'w', zipfile.ZIP_STORED) as zf:
for n in sorted(files, key=self.b):
with open(n, 'rb') as f:
zf.writestr(os.path.basename(n), f.read())
dest = self.j(self.RESOURCES, 'ebook-convert-complete.pickle')
files = []
for x in os.walk(self.j(self.SRC, 'calibre')):
for f in x[-1]:
if f.endswith('.py'):
files.append(self.j(x[0], f))
if self.newer(dest, files):
self.info('\tCreating ebook-convert-complete.pickle')
complete = {}
from calibre.ebooks.conversion.plumber import supported_input_formats
complete['input_fmts'] = set(supported_input_formats())
from calibre.web.feeds.recipes.collection import get_builtin_recipe_titles
complete['input_recipes'] = [
t + '.recipe ' for t in get_builtin_recipe_titles()
]
from calibre.customize.ui import available_output_formats
complete['output'] = set(available_output_formats())
from calibre.ebooks.conversion.cli import create_option_parser
from calibre.utils.logging import Log
log = Log()
#log.outputs = []
for inf in supported_input_formats():
if inf in ('zip', 'rar', 'oebzip'):
continue
for ouf in available_output_formats():
of = ouf if ouf == 'oeb' else 'dummy.' + ouf
p = create_option_parser(('ec', 'dummy1.' + inf, of, '-h'),
log)[0]
complete[(inf, ouf)] = [
x + ' ' for x in get_opts_from_parser(p)
]
cPickle.dump(complete, open(dest, 'wb'), -1)
self.info('\tCreating template-functions.json')
dest = self.j(self.RESOURCES, 'template-functions.json')
function_dict = {}
import inspect
from calibre.utils.formatter_functions import formatter_functions
for obj in formatter_functions().get_builtins().values():
eval_func = inspect.getmembers(
obj,
lambda x: inspect.ismethod(x) and x.__name__ == 'evaluate')
try:
lines = [
l[4:] for l in inspect.getsourcelines(eval_func[0][1])[0]
]
except:
continue
lines = ''.join(lines)
function_dict[obj.name] = lines
import json
json.dump(function_dict, open(dest, 'wb'), indent=4)
def catalog_option_parser(args):
from calibre.customize.ui import available_catalog_formats, plugin_for_catalog_format
from calibre.utils.logging import Log
def add_plugin_parser_options(fmt, parser, log):
# Fetch the extension-specific CLI options from the plugin
plugin = plugin_for_catalog_format(fmt)
for option in plugin.cli_options:
if option.action:
parser.add_option(option.option,
default=option.default,
dest=option.dest,
action=option.action,
help=option.help)
else:
parser.add_option(option.option,
default=option.default,
dest=option.dest,
help=option.help)
return plugin
def print_help(parser, log):
help = parser.format_help().encode(preferred_encoding, 'replace')
log(help)
def validate_command_line(parser, args, log):
# calibredb catalog path/to/destination.[epub|csv|xml|...] [options]
# Validate form
if not len(args) or args[0].startswith('-'):
print_help(parser, log)
log.error("\n\nYou must specify a catalog output file of the form 'path/to/destination.extension'\n"
"To review options for an output format, type 'calibredb catalog <.extension> --help'\n"
"For example, 'calibredb catalog .xml --help'\n")
raise SystemExit(1)
# Validate plugin exists for specified output format
output = os.path.abspath(args[0])
file_extension = output[output.rfind('.') + 1:].lower()
if not file_extension in available_catalog_formats():
print_help(parser, log)
log.error("No catalog plugin available for extension '%s'.\n" % file_extension +
"Catalog plugins available for %s\n" % ', '.join(available_catalog_formats()) )
raise SystemExit(1)
return output, file_extension
# Entry point
log = Log()
parser = get_parser(_(
'''
%prog catalog /path/to/destination.(CSV|EPUB|MOBI|XML ...) [options]
Export a catalog in format specified by path/to/destination extension.
Options control how entries are displayed in the generated catalog ouput.
'''))
# Confirm that a plugin handler exists for specified output file extension
# Will raise SystemExit(1) if no plugin matching file_extension
output, fmt = validate_command_line(parser, args, log)
# Add options common to all catalog plugins
parser.add_option('-i', '--ids', default=None, dest='ids',
help=_("Comma-separated list of database IDs to catalog.\n"
"If declared, --search is ignored.\n"
"Default: all"))
parser.add_option('-s', '--search', default=None, dest='search_text',
help=_("Filter the results by the search query. "
"For the format of the search query, please see "
"the search-related documentation in the User Manual.\n"
"Default: no filtering"))
parser.add_option('-v','--verbose', default=False, action='store_true',
dest='verbose',
help=_('Show detailed output information. Useful for debugging'))
# Add options specific to fmt plugin
plugin = add_plugin_parser_options(fmt, parser, log)
return parser, plugin, log
def create_fetcher(options, image_map={}, log=None):
if log is None:
log = Log()
return RecursiveFetcher(options, log, image_map={})
def do_download_for_worker(book, options, notification=lambda x, y: x):
'''
Child job, to download story when run as a worker job
'''
from calibre_plugins.fanficfare_plugin import FanFicFareBase
fffbase = FanFicFareBase(options['plugin_path'])
with fffbase:
from calibre_plugins.fanficfare_plugin.dialogs import (
NotGoingToDownload, OVERWRITE, OVERWRITEALWAYS, UPDATE,
UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
from calibre_plugins.fanficfare_plugin.fanficfare import adapters, writers, exceptions
from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import get_update_data
from calibre_plugins.fanficfare_plugin.fff_util import (
get_fff_adapter, get_fff_config)
try:
book['comment'] = _('Download started...')
configuration = get_fff_config(book['url'], options['fileform'],
options['personal.ini'])
if configuration.getConfig('use_ssl_unverified_context'):
## monkey patch to avoid SSL bug. dupliated from
## fff_plugin.py because bg jobs run in own process
## space.
import ssl
if hasattr(ssl, '_create_unverified_context'):
ssl._create_default_https_context = ssl._create_unverified_context
if not options[
'updateepubcover'] and 'epub_for_update' in book and options[
'collision'] in (UPDATE, UPDATEALWAYS):
configuration.set("overrides", "never_make_cover", "true")
# images only for epub, html, even if the user mistakenly
# turned it on else where.
if options['fileform'] not in ("epub", "html"):
configuration.set("overrides", "include_images", "false")
adapter = adapters.getAdapter(configuration, book['url'])
adapter.is_adult = book['is_adult']
adapter.username = book['username']
adapter.password = book['password']
adapter.setChaptersRange(book['begin'], book['end'])
adapter.load_cookiejar(options['cookiejarfile'])
logger.debug("cookiejar:%s" % adapter.cookiejar)
adapter.set_pagecache(options['pagecache'])
story = adapter.getStoryMetadataOnly()
if 'calibre_series' in book:
adapter.setSeries(book['calibre_series'][0],
book['calibre_series'][1])
# set PI version instead of default.
if 'version' in options:
story.setMetadata('version', options['version'])
writer = writers.getWriter(options['fileform'], configuration,
adapter)
outfile = book['outfile']
## No need to download at all. Shouldn't ever get down here.
if options['collision'] in (CALIBREONLY):
logger.info(
"Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening..."
)
book['comment'] = 'Metadata collected.'
## checks were done earlier, it's new or not dup or newer--just write it.
elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
('epub_for_update' not in book and options['collision'] in (UPDATE, UPDATEALWAYS)):
# preserve logfile even on overwrite.
if 'epub_for_update' in book:
(
urlignore,
chaptercountignore,
oldchaptersignore,
oldimgsignore,
oldcoverignore,
calibrebookmarkignore,
# only logfile set in adapter, so others aren't used.
adapter.logfile) = get_update_data(
book['epub_for_update'])
# change the existing entries id to notid so
# write_epub writes a whole new set to indicate overwrite.
if adapter.logfile:
adapter.logfile = adapter.logfile.replace(
"span id", "span notid")
logger.info("write to %s" % outfile)
writer.writeStory(outfilename=outfile, forceOverwrite=True)
book['comment'] = 'Download %s completed, %s chapters.' % (
options['fileform'], story.getMetadata("numChapters"))
## checks were done earlier, just update it.
elif 'epub_for_update' in book and options['collision'] in (
UPDATE, UPDATEALWAYS):
# update now handled by pre-populating the old images and
# chapters in the adapter rather than merging epubs.
urlchaptercount = int(
story.getMetadata('numChapters').replace(',', ''))
(url, chaptercount, adapter.oldchapters, adapter.oldimgs,
adapter.oldcover, adapter.calibrebookmark,
adapter.logfile) = get_update_data(book['epub_for_update'])
# dup handling from fff_plugin needed for anthology updates.
if options['collision'] == UPDATE:
if chaptercount == urlchaptercount:
book['comment'] = _(
"Already contains %d chapters. Reuse as is."
) % chaptercount
book['outfile'] = book[
'epub_for_update'] # for anthology merge ops.
return book
# dup handling from fff_plugin needed for anthology updates.
if chaptercount > urlchaptercount:
raise NotGoingToDownload(
_("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update."
) % (chaptercount, urlchaptercount),
'dialog_error.png')
if not (options['collision'] == UPDATEALWAYS and chaptercount == urlchaptercount) \
and adapter.getConfig("do_update_hook"):
chaptercount = adapter.hookForUpdates(chaptercount)
logger.info("Do update - epub(%d) vs url(%d)" %
(chaptercount, urlchaptercount))
logger.info("write to %s" % outfile)
writer.writeStory(outfilename=outfile, forceOverwrite=True)
book['comment'] = _('Update %s completed, added %s chapters for %s total.')%\
(options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
if options['smarten_punctuation'] and options['fileform'] == "epub" \
and calibre_version >= (0, 9, 39):
# for smarten punc
from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
from calibre.utils.logging import Log
from collections import namedtuple
# do smarten_punctuation from calibre's polish feature
data = {'smarten_punctuation': True}
opts = ALL_OPTS.copy()
opts.update(data)
O = namedtuple('Options', ' '.join(ALL_OPTS.iterkeys()))
opts = O(**opts)
log = Log(level=Log.DEBUG)
# report = []
polish({outfile: outfile}, opts, log,
logger.info) # report.append
except NotGoingToDownload as d:
book['good'] = False
book['comment'] = unicode(d)
book['icon'] = d.icon
except Exception as e:
book['good'] = False
book['comment'] = unicode(e)
book['icon'] = 'dialog_error.png'
book['status'] = 'Error'
logger.info("Exception: %s:%s" % (book, unicode(e)))
traceback.print_exc()
#time.sleep(10)
return book
def get_metadata(stream):
from calibre.ebooks.metadata import MetaInformation
from calibre.ptempfile import TemporaryDirectory
from calibre.ebooks.mobi.reader.headers import MetadataHeader
from calibre.ebooks.mobi.reader.mobi6 import MobiReader
from calibre import CurrentDir
try:
from PIL import Image as PILImage
PILImage
except ImportError:
import Image as PILImage
stream.seek(0)
try:
raw = stream.read(3)
except:
raw = ''
stream.seek(0)
if raw == b'TPZ':
from calibre.ebooks.metadata.topaz import get_metadata
return get_metadata(stream)
from calibre.utils.logging import Log
log = Log()
try:
mi = MetaInformation(os.path.basename(stream.name), [_('Unknown')])
except:
mi = MetaInformation(_('Unknown'), [_('Unknown')])
mh = MetadataHeader(stream, log)
if mh.title and mh.title != _('Unknown'):
mi.title = mh.title
if mh.exth is not None:
if mh.exth.mi is not None:
mi = mh.exth.mi
else:
size = 1024**3
if hasattr(stream, 'seek') and hasattr(stream, 'tell'):
pos = stream.tell()
stream.seek(0, 2)
size = stream.tell()
stream.seek(pos)
if size < 4 * 1024 * 1024:
with TemporaryDirectory('_mobi_meta_reader') as tdir:
with CurrentDir(tdir):
mr = MobiReader(stream, log)
parse_cache = {}
mr.extract_content(tdir, parse_cache)
if mr.embedded_mi is not None:
mi = mr.embedded_mi
if hasattr(mh.exth, 'cover_offset'):
cover_index = mh.first_image_index + mh.exth.cover_offset
data = mh.section_data(int(cover_index))
else:
try:
data = mh.section_data(mh.first_image_index)
except:
data = ''
buf = cStringIO.StringIO(data)
try:
im = PILImage.open(buf)
except:
log.exception('Failed to read MOBI cover')
else:
obuf = cStringIO.StringIO()
im.convert('RGB').save(obuf, format='JPEG')
mi.cover_data = ('jpg', obuf.getvalue())
return mi
def setup_pipeline(self, *args):
oidx = self.groups.currentIndex().row()
input_format = self.input_format
output_format = self.output_format
output_path = 'dummy.' + output_format
log = Log()
log.outputs = []
input_file = 'dummy.' + input_format
if input_format in ARCHIVE_FMTS:
input_file = 'dummy.html'
self.plumber = Plumber(input_file, output_path, log)
def widget_factory(cls):
return cls(self.stack, self.plumber.get_option_by_name,
self.plumber.get_option_help, self.db, self.book_id)
self.mw = widget_factory(MetadataWidget)
self.setWindowTitle(_('Convert') + ' ' + unicode(self.mw.title.text()))
lf = widget_factory(LookAndFeelWidget)
hw = widget_factory(HeuristicsWidget)
sr = widget_factory(SearchAndReplaceWidget)
ps = widget_factory(PageSetupWidget)
sd = widget_factory(StructureDetectionWidget)
toc = widget_factory(TOCWidget)
from calibre.gui2.actions.toc_edit import SUPPORTED
toc.manually_fine_tune_toc.setVisible(
output_format.upper() in SUPPORTED)
debug = widget_factory(DebugWidget)
output_widget = self.plumber.output_plugin.gui_configuration_widget(
self.stack, self.plumber.get_option_by_name,
self.plumber.get_option_help, self.db, self.book_id)
input_widget = self.plumber.input_plugin.gui_configuration_widget(
self.stack, self.plumber.get_option_by_name,
self.plumber.get_option_help, self.db, self.book_id)
while True:
c = self.stack.currentWidget()
if not c:
break
self.stack.removeWidget(c)
widgets = [self.mw, lf, hw, ps, sd, toc, sr]
if input_widget is not None:
widgets.append(input_widget)
if output_widget is not None:
widgets.append(output_widget)
widgets.append(debug)
for w in widgets:
self.stack.addWidget(w)
w.set_help_signal.connect(self.help.setPlainText)
self._groups_model = GroupModel(widgets)
self.groups.setModel(self._groups_model)
idx = oidx if -1 < oidx < self._groups_model.rowCount() else 0
self.groups.setCurrentIndex(self._groups_model.index(idx))
self.stack.setCurrentIndex(idx)
try:
shutil.rmtree(self.plumber.archive_input_tdir, ignore_errors=True)
except:
pass
import json
from calibre.utils.logging import Log
from calibre.ebooks.metadata.book.base import Metadata
from calibre_plugins.crossref_doi_download import DoiMeta
from calibre_plugins.crossref_doi_download.doi_reader import DoiReader, get_title, get_author_list
reader = DoiReader(Log())
dm = DoiMeta('./plugin/')
br = dm.browser
fullurl = 'https://api.crossref.org/works?query.bibliographic=Bayesian+data+analysis+Andrew+Gelman&mailto=vikoya5988%40oniaj.com'
cdata = br.open(fullurl).read()
data = json.loads(cdata)
message = data['message']
results = message['items']
identifiers = {}
# fin = map(lambda x:reader.result2meta(x,identifiers),results)
for r in results:
reader.result2meta(r,identifiers)
result = results[1]
title = get_title(result)
authors = get_author_list(result)
mi = Metadata(title=title, authors=authors)
from calibre import ipython
ipython(locals())
def get_metadata(stream):
from calibre.ebooks.metadata import MetaInformation
from calibre.ptempfile import TemporaryDirectory
from calibre.ebooks.mobi.reader.headers import MetadataHeader
from calibre.ebooks.mobi.reader.mobi6 import MobiReader
from calibre.utils.magick.draw import save_cover_data_to
from calibre import CurrentDir
stream.seek(0)
try:
raw = stream.read(3)
except:
raw = ''
stream.seek(0)
if raw == b'TPZ':
from calibre.ebooks.metadata.topaz import get_metadata
return get_metadata(stream)
from calibre.utils.logging import Log
log = Log()
try:
mi = MetaInformation(os.path.basename(stream.name), [_('Unknown')])
except:
mi = MetaInformation(_('Unknown'), [_('Unknown')])
mh = MetadataHeader(stream, log)
if mh.title and mh.title != _('Unknown'):
mi.title = mh.title
if mh.exth is not None:
if mh.exth.mi is not None:
mi = mh.exth.mi
else:
size = 1024**3
if hasattr(stream, 'seek') and hasattr(stream, 'tell'):
pos = stream.tell()
stream.seek(0, 2)
size = stream.tell()
stream.seek(pos)
if size < 4 * 1024 * 1024:
with TemporaryDirectory('_mobi_meta_reader') as tdir:
with CurrentDir(tdir):
mr = MobiReader(stream, log)
parse_cache = {}
mr.extract_content(tdir, parse_cache)
if mr.embedded_mi is not None:
mi = mr.embedded_mi
if hasattr(mh.exth, 'cover_offset'):
cover_index = mh.first_image_index + mh.exth.cover_offset
data = mh.section_data(int(cover_index))
else:
try:
data = mh.section_data(mh.first_image_index)
except:
data = ''
if data and what(None,
data) in {'jpg', 'jpeg', 'gif', 'png', 'bmp', 'webp'}:
try:
mi.cover_data = ('jpg',
save_cover_data_to(data,
'cover.jpg',
return_data=True))
except Exception:
log.exception('Failed to read MOBI cover')
return mi