def test_msgpack(self):
from calibre.utils.serialize import msgpack_dumps, msgpack_loads
from calibre.utils.date import utcnow
for obj in ({1:1}, utcnow()):
s = msgpack_dumps(obj)
self.assertEqual(obj, msgpack_loads(s))
self.assertEqual(type(msgpack_loads(msgpack_dumps(b'b'))), bytes)
self.assertEqual(type(msgpack_loads(msgpack_dumps(u'b'))), type(u''))
def test_msgpack(self):
from calibre.utils.serialize import msgpack_dumps, msgpack_loads
from calibre.utils.date import utcnow
for obj in ({1: 1}, utcnow()):
s = msgpack_dumps(obj)
self.assertEqual(obj, msgpack_loads(s))
self.assertEqual(type(msgpack_loads(msgpack_dumps(b'b'))), bytes)
self.assertEqual(type(msgpack_loads(msgpack_dumps(u'b'))), type(u''))
def test_msgpack(self):
from calibre.utils.serialize import msgpack_dumps, msgpack_loads
from calibre.utils.date import utcnow
for obj in ({1:1}, utcnow()):
s = msgpack_dumps(obj)
self.assertEqual(obj, msgpack_loads(s))
self.assertEqual(type(msgpack_loads(msgpack_dumps(b'b'))), bytes)
self.assertEqual(type(msgpack_loads(msgpack_dumps('b'))), unicode_type)
large = b'x' * (100 * 1024 * 1024)
msgpack_loads(msgpack_dumps(large))
def test_msgpack(self):
from calibre.utils.serialize import msgpack_dumps, msgpack_loads
from calibre.utils.date import utcnow
for obj in ({1:1}, utcnow()):
s = msgpack_dumps(obj)
self.assertEqual(obj, msgpack_loads(s))
self.assertEqual(type(msgpack_loads(msgpack_dumps(b'b'))), bytes)
self.assertEqual(type(msgpack_loads(msgpack_dumps(u'b'))), unicode_type)
large = b'x' * (100 * 1024 * 1024)
msgpack_loads(msgpack_dumps(large))
def __init__(self):
from calibre.utils.serialize import msgpack_loads
if self.kanwadict is None:
self.kanwadict = msgpack_loads(
P('localization/pykakasi/kanwadict2.calibre_msgpack', data=True))
if self.itaijidict is None:
self.itaijidict = msgpack_loads(
P('localization/pykakasi/itaijidict2.calibre_msgpack', data=True))
if self.kanadict is None:
self.kanadict = msgpack_loads(
P('localization/pykakasi/kanadict2.calibre_msgpack', data=True))
def __init__(self):
from calibre.utils.serialize import msgpack_loads
if self.kanwadict is None:
self.kanwadict = msgpack_loads(
P('localization/pykakasi/kanwadict2.calibre_msgpack', data=True))
if self.itaijidict is None:
self.itaijidict = msgpack_loads(
P('localization/pykakasi/itaijidict2.calibre_msgpack', data=True))
if self.kanadict is None:
self.kanadict = msgpack_loads(
P('localization/pykakasi/kanadict2.calibre_msgpack', data=True))
def get_codes():
global ccodes, ccodemap, country_names
if ccodes is None:
from calibre.utils.serialize import msgpack_loads
data = msgpack_loads(P('localization/iso3166.calibre_msgpack', allow_user_override=False, data=True))
ccodes, ccodemap, country_names = data['codes'], data['three_map'], data['names']
return ccodes, ccodemap
def cdb_run(ctx, rd, which, version):
try:
m = module_for_cmd(which)
except ImportError:
raise HTTPNotFound('No module named: {}'.format(which))
if not getattr(m, 'readonly', False):
ctx.check_for_write_access(rd)
if getattr(m, 'version', 0) != int(version):
raise HTTPNotFound(('The module {} is not available in version: {}.'
'Make sure the version of calibre used for the'
' server and calibredb match').format(which, version))
db = get_library_data(ctx, rd, strict_library_id=True)[0]
if ctx.restriction_for(rd, db):
raise HTTPForbidden('Cannot use the command-line db interface with a user who has per library restrictions')
raw = rd.read()
ct = rd.inheaders.get('Content-Type', all=True)
try:
if MSGPACK_MIME in ct:
args = msgpack_loads(raw)
elif 'application/json' in ct:
args = json_loads(raw)
else:
raise HTTPBadRequest('Only JSON or msgpack requests are supported')
except Exception:
raise HTTPBadRequest('args are not valid encoded data')
if getattr(m, 'needs_srv_ctx', False):
args = [ctx] + list(args)
try:
result = m.implementation(db, partial(ctx.notify_changes, db.backend.library_path), *args)
except Exception as err:
import traceback
return {'err': as_unicode(err), 'tb': traceback.format_exc()}
return {'result': result}
def _load_iso639():
global _iso639
if _iso639 is None:
ip = P('localization/iso639.calibre_msgpack', allow_user_override=False, data=True)
from calibre.utils.serialize import msgpack_loads
_iso639 = msgpack_loads(ip)
return _iso639
def do_print():
from calibre.customize.ui import plugin_for_input_format
stdin = getattr(sys.stdin, 'buffer', sys.stdin)
data = msgpack_loads(stdin.read())
ext = data['input'].lower().rpartition('.')[-1]
input_plugin = plugin_for_input_format(ext)
if input_plugin is None:
raise ValueError('Not a supported file type: {}'.format(ext.upper()))
args = [
'ebook-convert', data['input'], data['output'], '--paper-size',
data['paper_size'], '--pdf-add-toc', '--disable-remove-fake-margins',
'--chapter-mark', 'none', '-vv'
]
if input_plugin.is_image_collection:
args.append('--no-process')
else:
args.append('--disable-font-rescaling')
args.append('--page-breaks-before=/')
if data['page_numbers']:
args.append('--pdf-page-numbers')
for edge in 'left top right bottom'.split():
args.append('--pdf-page-margin-' + edge), args.append(
'%.1f' % (data['margin_' + edge] * 72))
from calibre.ebooks.conversion.cli import main
main(args)
def run_main(func):
from multiprocessing.connection import Client
from contextlib import closing
stdin = getattr(sys.stdin, 'buffer', sys.stdin)
address, key = msgpack_loads(eintr_retry_call(stdin.read))
with closing(Client(address, authkey=key)) as conn:
raise SystemExit(func(conn))
def run_main(func):
from multiprocessing.connection import Client
stdin = getattr(sys.stdin, 'buffer', sys.stdin)
address, key = msgpack_loads(eintr_retry_call(stdin.read))
with closing(Client(address, authkey=key)) as control_conn, closing(
Client(address, authkey=key)) as data_conn:
func(control_conn, data_conn)
def main():
# The entry point for the simple worker process
address = msgpack_loads(from_hex_bytes(os.environ['CALIBRE_WORKER_ADDRESS']))
key = from_hex_bytes(os.environ['CALIBRE_WORKER_KEY'])
with closing(Client(address, authkey=key)) as conn:
args = eintr_retry_call(conn.recv)
try:
mod, func, args, kwargs, module_is_source_code = args
if module_is_source_code:
importlib.import_module('calibre.customize.ui') # Load plugins
mod = compile_code(mod)
func = mod[func]
else:
try:
mod = importlib.import_module(mod)
except ImportError:
importlib.import_module('calibre.customize.ui') # Load plugins
mod = importlib.import_module(mod)
func = getattr(mod, func)
res = {'result':func(*args, **kwargs)}
except:
res = {'tb': traceback.format_exc()}
try:
conn.send(res)
except:
# Maybe EINTR
conn.send(res)
def _load_iso639():
global _iso639
if _iso639 is None:
ip = P('localization/iso639.calibre_msgpack', allow_user_override=False, data=True)
from calibre.utils.serialize import msgpack_loads
_iso639 = msgpack_loads(ip)
return _iso639
def run_main(func):
from multiprocessing.connection import Client
from contextlib import closing
stdin = getattr(sys.stdin, 'buffer', sys.stdin)
address, key = msgpack_loads(eintr_retry_call(stdin.read))
with closing(Client(address, authkey=key)) as conn:
raise SystemExit(func(conn))
def set_translators():
global _lang_trans, lcdata
# To test different translations invoke as
# CALIBRE_OVERRIDE_LANG=de_DE.utf8 program
lang = get_lang()
t = buf = iso639 = None
if 'CALIBRE_TEST_TRANSLATION' in os.environ:
buf = load_po(
os.path.expanduser(os.environ['CALIBRE_TEST_TRANSLATION']))
if lang:
mpath = get_lc_messages_path(lang)
if buf is None and mpath and os.access(mpath + '.po', os.R_OK):
buf = load_po(mpath + '.po')
if mpath is not None:
from zipfile import ZipFile
with ZipFile(
P('localization/locales.zip', allow_user_override=False),
'r') as zf:
if buf is None:
buf = io.BytesIO(zf.read(mpath + '/messages.mo'))
if mpath == 'nds':
mpath = 'de'
isof = mpath + '/iso639.mo'
try:
iso639 = io.BytesIO(zf.read(isof))
except:
pass # No iso639 translations for this lang
if buf is not None:
from calibre.utils.serialize import msgpack_loads
try:
lcdata = msgpack_loads(
zf.read(mpath + '/lcdata.calibre_msgpack'))
except:
pass # No lcdata
if buf is not None:
t = GNUTranslations(buf)
if iso639 is not None:
iso639 = _lang_trans = GNUTranslations(iso639)
t.add_fallback(iso639)
if t is None:
t = NullTranslations()
try:
set_translators.lang = t.info().get('language')
except Exception:
pass
if is_py3:
t.install(names=('ngettext', ))
else:
t.install(unicode=True, names=('ngettext', ))
# Now that we have installed a translator, we have to retranslate the help
# for the global prefs object as it was instantiated in get_lang(), before
# the translator was installed.
from calibre.utils.config_base import prefs
prefs.retranslate_help()
def __call__(self, names, args, in_process_container):
num_workers = len(self.workers) + 1
if num_workers == 1:
return [process_book_files(names, *args, container=in_process_container)]
group_sz = int(ceil(len(names) / num_workers))
groups = tuple(grouper(group_sz, names))
for group, worker in zip(groups[:-1], self.workers):
worker.stdin.write(as_bytes(msgpack_dumps((worker.output_path, group,) + args)))
worker.stdin.flush(), worker.stdin.close()
worker.job_sent = True
for worker in self.workers:
if not hasattr(worker, 'job_sent'):
worker.stdin.write(b'_'), worker.stdin.flush(), worker.stdin.close()
error = None
results = [process_book_files(groups[-1], *args, container=in_process_container)]
for worker in self.workers:
if not hasattr(worker, 'job_sent'):
worker.wait()
continue
if worker.wait() != 0:
with lopen(worker.error_path, 'rb') as f:
error = f.read().decode('utf-8', 'replace')
else:
with lopen(worker.output_path, 'rb') as f:
results.append(msgpack_loads(f.read()))
if error is not None:
raise Exception('Render worker failed with error:\n' + error)
return results
def main():
# The entry point for the simple worker process
address = msgpack_loads(
from_hex_bytes(os.environ['CALIBRE_WORKER_ADDRESS']))
key = from_hex_bytes(os.environ['CALIBRE_WORKER_KEY'])
with closing(Client(address, authkey=key)) as conn:
args = eintr_retry_call(conn.recv)
try:
mod, func, args, kwargs, module_is_source_code = args
if module_is_source_code:
importlib.import_module('calibre.customize.ui') # Load plugins
mod = compile_code(mod)
func = mod[func]
else:
try:
mod = importlib.import_module(mod)
except ImportError:
importlib.import_module(
'calibre.customize.ui') # Load plugins
mod = importlib.import_module(mod)
func = getattr(mod, func)
res = {'result': func(*args, **kwargs)}
except:
res = {'tb': traceback.format_exc()}
try:
conn.send(res)
except:
# Maybe EINTR
conn.send(res)
def offload():
# The entry point for the offload worker process
address = msgpack_loads(from_hex_bytes(os.environ['CALIBRE_WORKER_ADDRESS']))
key = from_hex_bytes(os.environ['CALIBRE_WORKER_KEY'])
func_cache = {}
with closing(Client(address, authkey=key)) as conn:
while True:
args = eintr_retry_call(conn.recv)
if args is None:
break
res = {'result':None, 'tb':None}
try:
mod, func, args, kwargs = args
if mod is None:
eintr_retry_call(conn.send, res)
continue
f = func_cache.get((mod, func), None)
if f is None:
try:
m = importlib.import_module(mod)
except ImportError:
importlib.import_module('calibre.customize.ui') # Load plugins
m = importlib.import_module(mod)
func_cache[(mod, func)] = f = getattr(m, func)
res['result'] = f(*args, **kwargs)
except:
import traceback
res['tb'] = traceback.format_exc()
eintr_retry_call(conn.send, res)
def get_codes():
global ccodes, ccodemap, country_names
if ccodes is None:
from calibre.utils.serialize import msgpack_loads
data = msgpack_loads(P('localization/iso3166.calibre_msgpack', allow_user_override=False, data=True))
ccodes, ccodemap, country_names = data['codes'], data['three_map'], data['names']
return ccodes, ccodemap
def offload():
# The entry point for the offload worker process
address = msgpack_loads(
from_hex_bytes(os.environ['CALIBRE_WORKER_ADDRESS']))
key = from_hex_bytes(os.environ['CALIBRE_WORKER_KEY'])
func_cache = {}
with closing(Client(address, authkey=key)) as conn:
while True:
args = eintr_retry_call(conn.recv)
if args is None:
break
res = {'result': None, 'tb': None}
try:
mod, func, args, kwargs = args
if mod is None:
eintr_retry_call(conn.send, res)
continue
f = func_cache.get((mod, func), None)
if f is None:
try:
m = importlib.import_module(mod)
except ImportError:
importlib.import_module(
'calibre.customize.ui') # Load plugins
m = importlib.import_module(mod)
func_cache[(mod, func)] = f = getattr(m, func)
res['result'] = f(*args, **kwargs)
except:
import traceback
res['tb'] = traceback.format_exc()
eintr_retry_call(conn.send, res)
def update_link_clicked(self, url):
url = str(url)
if url.startswith('update:'):
calibre_version, number_of_plugin_updates = msgpack_loads(
from_hex_bytes(url[len('update:'):]))
self.update_found(calibre_version,
number_of_plugin_updates,
force=True)
def update_link_clicked(self, url):
url = unicode_type(url)
if url.startswith('update:'):
calibre_version, number_of_plugin_updates = msgpack_loads(
binascii.unhexlify(url[len('update:'):]))
self.update_found(calibre_version,
number_of_plugin_updates,
force=True)
def set_translators():
global _lang_trans, lcdata
# To test different translations invoke as
# CALIBRE_OVERRIDE_LANG=de_DE.utf8 program
lang = get_lang()
t = buf = iso639 = None
if 'CALIBRE_TEST_TRANSLATION' in os.environ:
buf = load_po(os.path.expanduser(os.environ['CALIBRE_TEST_TRANSLATION']))
if lang:
mpath = get_lc_messages_path(lang)
if buf is None and mpath and os.access(mpath + '.po', os.R_OK):
buf = load_po(mpath + '.po')
if mpath is not None:
from zipfile import ZipFile
with ZipFile(P('localization/locales.zip',
allow_user_override=False), 'r') as zf:
if buf is None:
buf = io.BytesIO(zf.read(mpath + '/messages.mo'))
if mpath == 'nds':
mpath = 'de'
isof = mpath + '/iso639.mo'
try:
iso639 = io.BytesIO(zf.read(isof))
except:
pass # No iso639 translations for this lang
if buf is not None:
from calibre.utils.serialize import msgpack_loads
try:
lcdata = msgpack_loads(zf.read(mpath + '/lcdata.calibre_msgpack'))
except:
pass # No lcdata
if buf is not None:
t = GNUTranslations(buf)
if iso639 is not None:
iso639 = _lang_trans = GNUTranslations(iso639)
t.add_fallback(iso639)
if t is None:
t = NullTranslations()
try:
set_translators.lang = t.info().get('language')
except Exception:
pass
if is_py3:
t.install(names=('ngettext',))
else:
t.install(unicode=True, names=('ngettext',))
# Now that we have installed a translator, we have to retranslate the help
# for the global prefs object as it was instantiated in get_lang(), before
# the translator was installed.
from calibre.utils.config_base import prefs
prefs.retranslate_help()
def worker_main():
stdin = getattr(sys.stdin, 'buffer', sys.stdin)
raw = stdin.read()
if raw == b'_':
return
args = msgpack_loads(raw)
result = process_book_files(*args[1:])
with open(args[0], 'wb') as f:
f.write(as_bytes(msgpack_dumps(result)))
def available_translations():
global _available_translations
if _available_translations is None:
stats = P('localization/stats.calibre_msgpack', allow_user_override=False)
if os.path.exists(stats):
from calibre.utils.serialize import msgpack_loads
stats = msgpack_loads(open(stats, 'rb').read())
else:
stats = {}
_available_translations = [x for x in stats if stats[x] > 0.1]
return _available_translations
def available_translations():
global _available_translations
if _available_translations is None:
stats = P('localization/stats.calibre_msgpack', allow_user_override=False)
if os.path.exists(stats):
from calibre.utils.serialize import msgpack_loads
stats = msgpack_loads(open(stats, 'rb').read())
else:
stats = {}
_available_translations = [x for x in stats if stats[x] > 0.1]
return _available_translations
def cdb_set_fields(ctx, rd, book_id, library_id):
db = get_db(ctx, rd, library_id)
if ctx.restriction_for(rd, db):
raise HTTPForbidden(
'Cannot use the set fields interface with a user who has per library restrictions'
)
raw = rd.read()
ct = rd.inheaders.get('Content-Type', all=True)
ct = {x.lower().partition(';')[0] for x in ct}
try:
if MSGPACK_MIME in ct:
data = msgpack_loads(raw)
elif 'application/json' in ct:
data = json_loads(raw)
else:
raise HTTPBadRequest('Only JSON or msgpack requests are supported')
except Exception:
raise HTTPBadRequest('Invalid encoded data')
try:
changes, loaded_book_ids = data['changes'], frozenset(
map(int, data.get('loaded_book_ids', ())))
all_dirtied = bool(data.get('all_dirtied'))
if not isinstance(changes, dict):
raise TypeError('changes must be a dict')
except Exception:
raise HTTPBadRequest(
'''Data must be of the form {'changes': {'title': 'New Title', ...}, 'loaded_book_ids':[book_id1, book_id2, ...]'}'''
)
dirtied = set()
cdata = changes.pop('cover', False)
if cdata is not False:
if cdata is not None:
try:
cdata = standard_b64decode(
cdata.split(',', 1)[-1].encode('ascii'))
except Exception:
raise HTTPBadRequest(
'Cover data is not valid base64 encoded data')
try:
fmt = what(None, cdata)
except Exception:
fmt = None
if fmt not in ('jpeg', 'png'):
raise HTTPBadRequest('Cover data must be either JPEG or PNG')
dirtied |= db.set_cover({book_id: cdata})
for field, value in changes.iteritems():
dirtied |= db.set_field(field, {book_id: value})
ctx.notify_changes(db.backend.library_path, metadata(dirtied))
all_ids = dirtied if all_dirtied else (dirtied & loaded_book_ids)
all_ids |= {book_id}
return {bid: book_as_json(db, book_id) for bid in all_ids}
def load_payload_data(rd):
raw = rd.read()
ct = rd.inheaders.get('Content-Type', all=True)
ct = {x.lower().partition(';')[0] for x in ct}
try:
if MSGPACK_MIME in ct:
return msgpack_loads(raw)
elif 'application/json' in ct:
return json_loads(raw)
else:
raise HTTPBadRequest('Only JSON or msgpack requests are supported')
except Exception:
raise HTTPBadRequest('Invalid encoded data')
def load_payload_data(rd):
raw = rd.read()
ct = rd.inheaders.get('Content-Type', all=True)
ct = {x.lower().partition(';')[0] for x in ct}
try:
if MSGPACK_MIME in ct:
return msgpack_loads(raw)
elif 'application/json' in ct:
return json_loads(raw)
else:
raise HTTPBadRequest('Only JSON or msgpack requests are supported')
except Exception:
raise HTTPBadRequest('Invalid encoded data')
def translator_for_lang(lang):
t = buf = iso639 = lcdata = None
if 'CALIBRE_TEST_TRANSLATION' in os.environ:
buf = load_po(
os.path.expanduser(os.environ['CALIBRE_TEST_TRANSLATION']))
mpath = get_lc_messages_path(lang)
if buf is None and mpath and os.access(mpath + '.po', os.R_OK):
buf = load_po(mpath + '.po')
if mpath is not None:
from zipfile import ZipFile
with ZipFile(P('localization/locales.zip', allow_user_override=False),
'r') as zf:
if buf is None:
buf = io.BytesIO(zf.read(mpath + '/messages.mo'))
if mpath == 'nds':
mpath = 'de'
isof = mpath + '/iso639.mo'
try:
iso639 = io.BytesIO(zf.read(isof))
except:
pass # No iso639 translations for this lang
if buf is not None:
from calibre.utils.serialize import msgpack_loads
try:
lcdata = msgpack_loads(
zf.read(mpath + '/lcdata.calibre_msgpack'))
except:
pass # No lcdata
if buf is not None:
try:
t = GNUTranslations(buf)
except Exception:
import traceback
traceback.print_exc()
t = None
if iso639 is not None:
try:
iso639 = GNUTranslations(iso639)
except Exception:
iso639 = None
else:
if t is not None:
t.add_fallback(iso639)
if t is None:
t = NullTranslations()
return {'translator': t, 'iso639_translator': iso639, 'lcdata': lcdata}
def base_dir():
global _base_dir
if _base_dir is not None and not os.path.exists(_base_dir):
# Some people seem to think that running temp file cleaners that
# delete the temp dirs of running programs is a good idea!
_base_dir = None
if _base_dir is None:
td = os.environ.get('CALIBRE_WORKER_TEMP_DIR', None)
if td is not None:
from calibre.utils.serialize import msgpack_loads
from polyglot.binary import from_hex_bytes
try:
td = msgpack_loads(from_hex_bytes(td))
except Exception:
td = None
if td and os.path.exists(td):
_base_dir = td
else:
base = os.environ.get('CALIBRE_TEMP_DIR', None)
if base is not None and iswindows:
base = getenv('CALIBRE_TEMP_DIR')
prefix = app_prefix('tmp_')
if base is None:
if iswindows:
# On windows, if the TMP env var points to a path that
# cannot be encoded using the mbcs encoding, then the
# python 2 tempfile algorithm for getting the temporary
# directory breaks. So we use the win32 api to get a
# unicode temp path instead. See
# https://bugs.launchpad.net/bugs/937389
base = get_windows_temp_path()
elif isosx:
# Use the cache dir rather than the temp dir for temp files as Apple
# thinks deleting unused temp files is a good idea. See note under
# _CS_DARWIN_USER_TEMP_DIR here
# https://developer.apple.com/library/mac/documentation/Darwin/Reference/ManPages/man3/confstr.3.html
base = osx_cache_dir()
_base_dir = tempfile.mkdtemp(prefix=prefix, dir=base)
atexit.register(determined_remove_dir if iswindows else remove_dir,
_base_dir)
try:
tempfile.gettempdir()
except Exception:
# Widows temp vars set to a path not encodable in mbcs
# Use our temp dir
tempfile.tempdir = _base_dir
return _base_dir
def __init__(self, comp_line, pos):
words = split(comp_line[:pos])
char_before = comp_line[pos-1]
prefix = words[-1] if words[-1].endswith(char_before) else ''
wc = len(words)
if not prefix:
wc += 1
self.words = words
self.prefix = prefix
self.previous = words[-2 if prefix else -1]
from calibre.utils.serialize import msgpack_loads
self.cache = msgpack_loads(open(os.path.join(sys.resources_location,
'ebook-convert-complete.calibre_msgpack'), 'rb').read())
self.complete(wc)
def test_serialize_metadata(self): # {{{
from calibre.utils.serialize import json_dumps, json_loads, msgpack_dumps, msgpack_loads
from calibre.library.field_metadata import fm_as_dict
cache = self.init_cache(self.library_path)
fm = cache.field_metadata
for d, l in ((json_dumps, json_loads), (msgpack_dumps, msgpack_loads)):
fm2 = l(d(fm))
self.assertEqual(fm_as_dict(fm), fm_as_dict(fm2))
for i in range(1, 4):
mi = cache.get_metadata(i, get_cover=True, cover_as_data=True)
rmi = msgpack_loads(msgpack_dumps(mi))
self.compare_metadata(mi, rmi, exclude='format_metadata has_cover formats id'.split())
rmi = json_loads(json_dumps(mi))
self.compare_metadata(mi, rmi, exclude='format_metadata has_cover formats id'.split())
def __init__(self, comp_line, pos):
words = split(comp_line[:pos])
char_before = comp_line[pos-1]
prefix = words[-1] if words[-1].endswith(char_before) else ''
wc = len(words)
if not prefix:
wc += 1
self.words = words
self.prefix = prefix
self.previous = words[-2 if prefix else -1]
from calibre.utils.serialize import msgpack_loads
self.cache = msgpack_loads(open(os.path.join(sys.resources_location,
'ebook-convert-complete.calibre_msgpack'), 'rb').read(), use_list=False)
self.complete(wc)
def test_serialize_metadata(self): # {{{
from calibre.utils.serialize import json_dumps, json_loads, msgpack_dumps, msgpack_loads
from calibre.library.field_metadata import fm_as_dict
cache = self.init_cache(self.library_path)
fm = cache.field_metadata
for d, l in ((json_dumps, json_loads), (msgpack_dumps, msgpack_loads)):
fm2 = l(d(fm))
self.assertEqual(fm_as_dict(fm), fm_as_dict(fm2))
for i in range(1, 4):
mi = cache.get_metadata(i, get_cover=True, cover_as_data=True)
rmi = msgpack_loads(msgpack_dumps(mi))
self.compare_metadata(mi, rmi, exclude='format_metadata has_cover formats id'.split())
rmi = json_loads(json_dumps(mi))
self.compare_metadata(mi, rmi, exclude='format_metadata has_cover formats id'.split())
def load_jisyo(self, char):
if not isinstance(char, str):
char = str(char, 'utf-8')
key = "%04x"%ord(char)
try: # already exist?
table = self.jisyo_table[key]
except:
from calibre.utils.serialize import msgpack_loads
try:
table = self.jisyo_table[key] = msgpack_loads(decompress(self.kanwadict[key]))
except:
return None
return table
def load_jisyo(self, char):
if not isinstance(char, unicode_type):
char = unicode_type(char, 'utf-8')
key = "%04x"%ord(char)
try: # already exist?
table = self.jisyo_table[key]
except:
from calibre.utils.serialize import msgpack_loads
try:
table = self.jisyo_table[key] = msgpack_loads(decompress(self.kanwadict[key]))
except:
return None
return table
def base_dir():
global _base_dir
if _base_dir is not None and not os.path.exists(_base_dir):
# Some people seem to think that running temp file cleaners that
# delete the temp dirs of running programs is a good idea!
_base_dir = None
if _base_dir is None:
td = os.environ.get('CALIBRE_WORKER_TEMP_DIR', None)
if td is not None:
from calibre.utils.serialize import msgpack_loads
from polyglot.binary import from_hex_bytes
try:
td = msgpack_loads(from_hex_bytes(td))
except Exception:
td = None
if td and os.path.exists(td):
_base_dir = td
else:
base = os.environ.get('CALIBRE_TEMP_DIR', None)
if base is not None and iswindows:
base = getenv('CALIBRE_TEMP_DIR')
prefix = app_prefix('tmp_')
if base is None:
if iswindows:
# On windows, if the TMP env var points to a path that
# cannot be encoded using the mbcs encoding, then the
# python 2 tempfile algorithm for getting the temporary
# directory breaks. So we use the win32 api to get a
# unicode temp path instead. See
# https://bugs.launchpad.net/bugs/937389
base = get_windows_temp_path()
elif isosx:
# Use the cache dir rather than the temp dir for temp files as Apple
# thinks deleting unused temp files is a good idea. See note under
# _CS_DARWIN_USER_TEMP_DIR here
# https://developer.apple.com/library/mac/documentation/Darwin/Reference/ManPages/man3/confstr.3.html
base = osx_cache_dir()
_base_dir = tempfile.mkdtemp(prefix=prefix, dir=base)
atexit.register(determined_remove_dir if iswindows else remove_dir, _base_dir)
try:
tempfile.gettempdir()
except:
# Widows temp vars set to a path not encodable in mbcs
# Use our temp dir
tempfile.tempdir = _base_dir
return _base_dir
def load_jisyo(self, char):
try: # python2
key = "%04x"%ord(unicode(char))
except: # python3
key = "%04x"%ord(char)
try: # already exist?
table = self.jisyo_table[key]
except:
from calibre.utils.serialize import msgpack_loads
try:
table = self.jisyo_table[key] = msgpack_loads(decompress(self.kanwadict[key]))
except:
return None
return table
def remote_run(self, name, m, *args):
from mechanize import HTTPError, Request
from calibre.utils.serialize import msgpack_loads, msgpack_dumps
url = self.url + '/cdb/cmd/{}/{}'.format(name, getattr(m, 'version', 0))
if self.library_id:
url += '?' + urlencode({'library_id':self.library_id})
rq = Request(url, data=msgpack_dumps(args),
headers={'Accept': MSGPACK_MIME, 'Content-Type': MSGPACK_MIME})
try:
res = self.br.open_novisit(rq)
ans = msgpack_loads(res.read())
except HTTPError as err:
self.interpret_http_error(err)
raise
if 'err' in ans:
prints(ans['tb'])
raise SystemExit(ans['err'])
return ans['result']
def remote_run(self, name, m, *args):
from mechanize import HTTPError, Request
from calibre.utils.serialize import msgpack_loads, msgpack_dumps
url = self.url + '/cdb/cmd/{}/{}'.format(name, getattr(m, 'version', 0))
if self.library_id:
url += '?' + urlencode({'library_id':self.library_id})
rq = Request(url, data=msgpack_dumps(args),
headers={'Accept': MSGPACK_MIME, 'Content-Type': MSGPACK_MIME})
try:
res = self.br.open_novisit(rq, timeout=self.timeout)
ans = msgpack_loads(res.read())
except HTTPError as err:
self.interpret_http_error(err)
raise
if 'err' in ans:
if ans['tb']:
prints(ans['tb'])
raise SystemExit(ans['err'])
return ans['result']
def do_print():
from calibre.customize.ui import plugin_for_input_format
stdin = getattr(sys.stdin, 'buffer', sys.stdin)
data = msgpack_loads(stdin.read())
ext = data['input'].lower().rpartition('.')[-1]
input_plugin = plugin_for_input_format(ext)
args = ['ebook-convert', data['input'], data['output'], '--paper-size', data['paper_size'], '--pdf-add-toc',
'--disable-remove-fake-margins', '--chapter-mark', 'none', '-vv']
if input_plugin.is_image_collection:
args.append('--no-process')
else:
args.append('--disable-font-rescaling')
args.append('--page-breaks-before=/')
if data['page_numbers']:
args.append('--pdf-page-numbers')
for edge in 'left top right bottom'.split():
args.append('--pdf-page-margin-' + edge), args.append('%.1f' % (data['margin_' + edge] * 72))
from calibre.ebooks.conversion.cli import main
main(args)
def cdb_set_fields(ctx, rd, book_id, library_id):
db = get_db(ctx, rd, library_id)
if ctx.restriction_for(rd, db):
raise HTTPForbidden('Cannot use the set fields interface with a user who has per library restrictions')
raw = rd.read()
ct = rd.inheaders.get('Content-Type', all=True)
ct = {x.lower().partition(';')[0] for x in ct}
try:
if MSGPACK_MIME in ct:
data = msgpack_loads(raw)
elif 'application/json' in ct:
data = json_loads(raw)
else:
raise HTTPBadRequest('Only JSON or msgpack requests are supported')
changes, loaded_book_ids = data['changes'], frozenset(map(int, data['loaded_book_ids']))
except Exception:
raise HTTPBadRequest('Invalid encoded data')
dirtied = set()
for field, value in changes.iteritems():
dirtied |= db.set_field(field, {book_id: value})
metadata(dirtied)
return {bid: book_as_json(db, book_id) for bid in (dirtied & loaded_book_ids) | {book_id}}
def cdb_set_fields(ctx, rd, book_id, library_id):
db = get_db(ctx, rd, library_id)
if ctx.restriction_for(rd, db):
raise HTTPForbidden('Cannot use the set fields interface with a user who has per library restrictions')
raw = rd.read()
ct = rd.inheaders.get('Content-Type', all=True)
ct = {x.lower().partition(';')[0] for x in ct}
try:
if MSGPACK_MIME in ct:
data = msgpack_loads(raw)
elif 'application/json' in ct:
data = json_loads(raw)
else:
raise HTTPBadRequest('Only JSON or msgpack requests are supported')
changes, loaded_book_ids = data['changes'], frozenset(map(int, data['loaded_book_ids']))
except Exception:
raise HTTPBadRequest('Invalid encoded data')
dirtied = set()
cdata = changes.pop('cover', False)
if cdata is not False:
if cdata is not None:
try:
cdata = standard_b64decode(cdata.split(',', 1)[-1].encode('ascii'))
except Exception:
raise HTTPBadRequest('Cover data is not valid base64 encoded data')
try:
fmt = what(None, cdata)
except Exception:
fmt = None
if fmt not in ('jpeg', 'png'):
raise HTTPBadRequest('Cover data must be either JPEG or PNG')
dirtied |= db.set_cover({book_id: cdata})
for field, value in changes.iteritems():
dirtied |= db.set_field(field, {book_id: value})
ctx.notify_changes(db.backend.library_path, metadata(dirtied))
return {bid: book_as_json(db, book_id) for bid in (dirtied & loaded_book_ids) | {book_id}}
def __init__(self, opts, info=prints, warn=None, manifest=None):
self.opts = opts
self.info = info
self.warn = warn
self.warnings = []
if self.warn is None:
self.warn = self.warning
if not self.opts.staging_bindir:
self.opts.staging_bindir = os.path.join(self.opts.staging_root,
'bin')
if not self.opts.staging_sharedir:
self.opts.staging_sharedir = os.path.join(self.opts.staging_root,
'share', 'calibre')
self.opts.staging_etc = '/etc' if self.opts.staging_root == '/usr' else \
os.path.join(self.opts.staging_root, 'etc')
prefix = getattr(self.opts, 'prefix', None)
if prefix and prefix != self.opts.staging_root:
self.opts.staged_install = True
os.environ['XDG_DATA_DIRS'] = os.path.join(self.opts.staging_root, 'share')
os.environ['XDG_UTILS_INSTALL_MODE'] = 'system'
from calibre.utils.serialize import msgpack_loads
scripts = msgpack_loads(P('scripts.calibre_msgpack', data=True))
self.manifest = manifest or []
if getattr(sys, 'frozen_path', False):
if os.access(self.opts.staging_bindir, os.W_OK):
self.info('Creating symlinks...')
for exe in scripts.keys():
dest = os.path.join(self.opts.staging_bindir, exe)
if os.path.lexists(dest):
os.unlink(dest)
tgt = os.path.join(getattr(sys, 'frozen_path'), exe)
self.info('\tSymlinking %s to %s'%(tgt, dest))
os.symlink(tgt, dest)
self.manifest.append(dest)
else:
self.warning(textwrap.fill(
'No permission to write to %s, not creating program launch symlinks,'
' you should ensure that %s is in your PATH or create the symlinks yourself' % (
self.opts.staging_bindir, getattr(sys, 'frozen_path', 'the calibre installation directory'))))
self.icon_resources = []
self.menu_resources = []
self.mime_resources = []
self.appdata_resources = []
if islinux or isbsd:
self.setup_completion()
if islinux or isbsd:
self.setup_desktop_integration()
if not getattr(self.opts, 'staged_install', False):
self.create_uninstaller()
from calibre.utils.config import config_dir
if os.path.exists(config_dir):
os.chdir(config_dir)
if islinux or isbsd:
for f in os.listdir('.'):
if os.stat(f).st_uid == 0:
import shutil
shutil.rmtree(f) if os.path.isdir(f) else os.unlink(f)
if os.stat(config_dir).st_uid == 0:
os.rmdir(config_dir)
if warn is None and self.warnings:
self.info('\n\nThere were %d warnings\n'%len(self.warnings))
for args, kwargs in self.warnings:
self.info('*', *args, **kwargs)
print()
def main():
if iswindows:
if '--multiprocessing-fork' in sys.argv:
# We are using the multiprocessing module on windows to launch a
# worker process
from multiprocessing import freeze_support
freeze_support()
return 0
# Close open file descriptors inherited from parent
# On Unix this is done by the subprocess module
os.closerange(3, 256)
if isosx and 'CALIBRE_WORKER_ADDRESS' not in os.environ and 'CALIBRE_SIMPLE_WORKER' not in os.environ and '--pipe-worker' not in sys.argv:
# On some OS X computers launchd apparently tries to
# launch the last run process from the bundle
# so launch the gui as usual
from calibre.gui2.main import main as gui_main
return gui_main(['calibre'])
csw = os.environ.get('CALIBRE_SIMPLE_WORKER', None)
if csw:
mod, _, func = csw.partition(':')
mod = importlib.import_module(mod)
func = getattr(mod, func)
func()
return
if '--pipe-worker' in sys.argv:
try:
exec (sys.argv[-1])
except Exception:
print('Failed to run pipe worker with command:', sys.argv[-1])
raise
return
address = msgpack_loads(from_hex_bytes(os.environ['CALIBRE_WORKER_ADDRESS']))
key = from_hex_bytes(os.environ['CALIBRE_WORKER_KEY'])
resultf = from_hex_unicode(os.environ['CALIBRE_WORKER_RESULT'])
with closing(Client(address, authkey=key)) as conn:
name, args, kwargs, desc = eintr_retry_call(conn.recv)
if desc:
prints(desc)
sys.stdout.flush()
func, notification = get_func(name)
notifier = Progress(conn)
if notification:
kwargs[notification] = notifier
notifier.start()
result = func(*args, **kwargs)
if result is not None and os.path.exists(os.path.dirname(resultf)):
with lopen(resultf, 'wb') as f:
f.write(pickle_dumps(result))
notifier.queue.put(None)
try:
sys.stdout.flush()
except EnvironmentError:
pass # Happens sometimes on OS X for GUI processes (EPIPE)
try:
sys.stderr.flush()
except EnvironmentError:
pass # Happens sometimes on OS X for GUI processes (EPIPE)
return 0
def __init__(self, opts, info=prints, warn=None, manifest=None):
self.opts = opts
self.info = info
self.warn = warn
self.warnings = []
if self.warn is None:
self.warn = self.warning
if not self.opts.staging_bindir:
self.opts.staging_bindir = os.path.join(self.opts.staging_root,
'bin')
if not self.opts.staging_sharedir:
self.opts.staging_sharedir = os.path.join(self.opts.staging_root,
'share', 'calibre')
self.opts.staging_etc = '/etc' if self.opts.staging_root == '/usr' else \
os.path.join(self.opts.staging_root, 'etc')
from calibre.utils.serialize import msgpack_loads
scripts = msgpack_loads(P('scripts.calibre_msgpack', data=True))
self.manifest = manifest or []
if getattr(sys, 'frozen_path', False):
if os.access(self.opts.staging_bindir, os.W_OK):
self.info('Creating symlinks...')
for exe in scripts.keys():
dest = os.path.join(self.opts.staging_bindir, exe)
if os.path.lexists(dest):
os.unlink(dest)
tgt = os.path.join(getattr(sys, 'frozen_path'), exe)
self.info('\tSymlinking %s to %s' % (tgt, dest))
os.symlink(tgt, dest)
self.manifest.append(dest)
else:
self.warning(
textwrap.fill(
'No permission to write to %s, not creating program launch symlinks,'
' you should ensure that %s is in your PATH or create the symlinks yourself'
% (self.opts.staging_bindir,
getattr(sys, 'frozen_path',
'the calibre installation directory'))))
self.icon_resources = []
self.menu_resources = []
self.mime_resources = []
self.appdata_resources = []
if islinux or isbsd:
self.setup_completion()
if islinux or isbsd:
self.setup_desktop_integration()
self.create_uninstaller()
from calibre.utils.config import config_dir
if os.path.exists(config_dir):
os.chdir(config_dir)
if islinux or isbsd:
for f in os.listdir('.'):
if os.stat(f).st_uid == 0:
import shutil
shutil.rmtree(f) if os.path.isdir(f) else os.unlink(f)
if os.stat(config_dir).st_uid == 0:
os.rmdir(config_dir)
if warn is None and self.warnings:
self.info('\n\nThere were %d warnings\n' % len(self.warnings))
for args, kwargs in self.warnings:
self.info('*', *args, **kwargs)
print()
def run_main(func):
from multiprocessing.connection import Client
stdin = getattr(sys.stdin, 'buffer', sys.stdin)
address, key = msgpack_loads(eintr_retry_call(stdin.read))
with closing(Client(address, authkey=key)) as control_conn, closing(Client(address, authkey=key)) as data_conn:
func(control_conn, data_conn)
def viewer_main():
stdin = getattr(sys.stdin, 'buffer', sys.stdin)
args = msgpack_loads(stdin.read())
render_for_viewer(*args)
def main():
if iswindows:
if '--multiprocessing-fork' in sys.argv:
# We are using the multiprocessing module on windows to launch a
# worker process
from multiprocessing import freeze_support
freeze_support()
return 0
# Close open file descriptors inherited from parent
# On Unix this is done by the subprocess module
os.closerange(3, 256)
if isosx and 'CALIBRE_WORKER_ADDRESS' not in os.environ and 'CALIBRE_SIMPLE_WORKER' not in os.environ and '--pipe-worker' not in sys.argv:
# On some OS X computers launchd apparently tries to
# launch the last run process from the bundle
# so launch the gui as usual
from calibre.gui2.main import main as gui_main
return gui_main(['calibre'])
csw = os.environ.get('CALIBRE_SIMPLE_WORKER', None)
if csw:
mod, _, func = csw.partition(':')
mod = importlib.import_module(mod)
func = getattr(mod, func)
func()
return
if '--pipe-worker' in sys.argv:
try:
exec(sys.argv[-1])
except Exception:
print('Failed to run pipe worker with command:', sys.argv[-1])
raise
return
address = msgpack_loads(from_hex_bytes(os.environ['CALIBRE_WORKER_ADDRESS']))
key = from_hex_bytes(os.environ['CALIBRE_WORKER_KEY'])
resultf = from_hex_unicode(os.environ['CALIBRE_WORKER_RESULT'])
with closing(Client(address, authkey=key)) as conn:
name, args, kwargs, desc = eintr_retry_call(conn.recv)
if desc:
prints(desc)
sys.stdout.flush()
func, notification = get_func(name)
notifier = Progress(conn)
if notification:
kwargs[notification] = notifier
notifier.start()
result = func(*args, **kwargs)
if result is not None and os.path.exists(os.path.dirname(resultf)):
with lopen(resultf, 'wb') as f:
f.write(pickle_dumps(result))
notifier.queue.put(None)
try:
sys.stdout.flush()
except EnvironmentError:
pass # Happens sometimes on OS X for GUI processes (EPIPE)
try:
sys.stderr.flush()
except EnvironmentError:
pass # Happens sometimes on OS X for GUI processes (EPIPE)
return 0
