def kanwaout(self, out):
from calibre.utils.serialize import msgpack_dumps
with open(out, 'wb') as f:
dic = {}
for k, v in iteritems(self.records):
dic[k] = compress(msgpack_dumps(v))
f.write(msgpack_dumps(dic))
def kanwaout(self, out):
from calibre.utils.serialize import msgpack_dumps
with open(out, 'wb') as f:
dic = {}
for k, v in iteritems(self.records):
dic[k] = compress(msgpack_dumps(v))
f.write(msgpack_dumps(dic))
def test_msgpack(self):
from calibre.utils.serialize import msgpack_dumps, msgpack_loads
from calibre.utils.date import utcnow
for obj in ({1:1}, utcnow()):
s = msgpack_dumps(obj)
self.assertEqual(obj, msgpack_loads(s))
self.assertEqual(type(msgpack_loads(msgpack_dumps(b'b'))), bytes)
self.assertEqual(type(msgpack_loads(msgpack_dumps(u'b'))), type(u''))
def test_msgpack(self):
from calibre.utils.serialize import msgpack_dumps, msgpack_loads
from calibre.utils.date import utcnow
for obj in ({1: 1}, utcnow()):
s = msgpack_dumps(obj)
self.assertEqual(obj, msgpack_loads(s))
self.assertEqual(type(msgpack_loads(msgpack_dumps(b'b'))), bytes)
self.assertEqual(type(msgpack_loads(msgpack_dumps(u'b'))), type(u''))
def run(self, opts):
from calibre.utils.serialize import msgpack_dumps
scripts = {}
for x in ['console']:
for name in basenames[x]:
if name in ('calibre-complete', 'calibre_postinstall'):
continue
scripts[name] = x
dest = self.j(self.RESOURCES, 'scripts.calibre_msgpack')
if self.newer(dest, self.j(self.SRC, 'calibre', 'linux.py')):
self.info('\tCreating ' + self.b(dest))
with open(dest, 'wb') as f:
f.write(msgpack_dumps(scripts))
recipe_icon_dir = self.a(
self.j(self.RESOURCES, '..', 'recipes', 'icons'))
dest = os.path.splitext(dest)[0] + '.zip'
files = glob.glob(self.j(recipe_icon_dir, '*.png'))
if self.newer(dest, files):
self.info('\tCreating builtin_recipes.zip')
with zipfile.ZipFile(dest, 'w', zipfile.ZIP_STORED) as zf:
for n in sorted(files, key=self.b):
with open(n, 'rb') as f:
zf.writestr(self.b(n), f.read())
dest = self.j(self.RESOURCES, 'ebook-convert-complete.calibre_msgpack')
files = []
for x in os.walk(self.j(self.SRC, 'calibre')):
for f in x[-1]:
if f.endswith('.py'):
files.append(self.j(x[0], f))
if self.newer(dest, files):
self.info('\tCreating ' + self.b(dest))
complete = {}
from calibre.ebooks.conversion.plumber import supported_input_formats
complete['input_fmts'] = set(supported_input_formats())
from calibre.customize.ui import available_output_formats
complete['output'] = set(available_output_formats())
from calibre.ebooks.conversion.cli import create_option_parser
from calibre.utils.logging import Log
log = Log()
# log.outputs = []
for inf in supported_input_formats():
if inf in ('zip', 'rar', 'oebzip'):
continue
for ouf in available_output_formats():
of = ouf if ouf == 'oeb' else 'dummy.' + ouf
p = create_option_parser(('ec', 'dummy1.' + inf, of, '-h'),
log)[0]
complete[(inf, ouf)] = [
x + ' ' for x in get_opts_from_parser(p)
]
with open(dest, 'wb') as f:
f.write(msgpack_dumps(only_unicode_recursive(complete)))
def test_msgpack(self):
from calibre.utils.serialize import msgpack_dumps, msgpack_loads
from calibre.utils.date import utcnow
for obj in ({1:1}, utcnow()):
s = msgpack_dumps(obj)
self.assertEqual(obj, msgpack_loads(s))
self.assertEqual(type(msgpack_loads(msgpack_dumps(b'b'))), bytes)
self.assertEqual(type(msgpack_loads(msgpack_dumps(u'b'))), unicode_type)
large = b'x' * (100 * 1024 * 1024)
msgpack_loads(msgpack_dumps(large))
def test_msgpack(self):
from calibre.utils.serialize import msgpack_dumps, msgpack_loads
from calibre.utils.date import utcnow
for obj in ({1:1}, utcnow()):
s = msgpack_dumps(obj)
self.assertEqual(obj, msgpack_loads(s))
self.assertEqual(type(msgpack_loads(msgpack_dumps(b'b'))), bytes)
self.assertEqual(type(msgpack_loads(msgpack_dumps('b'))), unicode_type)
large = b'x' * (100 * 1024 * 1024)
msgpack_loads(msgpack_dumps(large))
def msgpack(ctx, rd, endpoint, output):
rd.outheaders.set('Content-Type', MSGPACK_MIME, replace_all=True)
if isinstance(output, bytes) or hasattr(output, 'fileno'):
ans = output # Assume output is already msgpack encoded
else:
ans = msgpack_dumps(output)
return ans
def run(self, opts):
src = self.j(self.d(self.SRC), 'setup', 'iso_639_3.xml')
if not os.path.exists(src):
raise Exception(src + ' does not exist')
dest = self.DEST
base = self.d(dest)
if not os.path.exists(base):
os.makedirs(base)
if not self.newer(dest, [src, __file__]):
self.info('Packed code is up to date')
return
self.info('Packing ISO-639 codes to', dest)
from lxml import etree
root = etree.fromstring(open(src, 'rb').read())
by_2 = {}
by_3b = {}
by_3t = {}
m2to3 = {}
m3to2 = {}
m3bto3t = {}
nm = {}
codes2, codes3t, codes3b = set(), set(), set()
unicode_type = type(u'')
for x in root.xpath('//iso_639_3_entry'):
two = x.get('part1_code', None)
if two:
two = unicode_type(two)
threet = x.get('id')
if threet:
threet = unicode_type(threet)
threeb = x.get('part2_code', None)
if threeb:
threeb = unicode_type(threeb)
if threeb is None:
# Only recognize languages in ISO-639-2
continue
name = x.get('name')
if name:
name = unicode_type(name)
if two is not None:
by_2[two] = name
codes2.add(two)
m2to3[two] = threet
m3to2[threeb] = m3to2[threet] = two
by_3b[threeb] = name
by_3t[threet] = name
if threeb != threet:
m3bto3t[threeb] = threet
codes3b.add(threeb)
codes3t.add(threet)
base_name = name.lower()
nm[base_name] = threet
x = {u'by_2':by_2, u'by_3b':by_3b, u'by_3t':by_3t, u'codes2':codes2,
u'codes3b':codes3b, u'codes3t':codes3t, u'2to3':m2to3,
u'3to2':m3to2, u'3bto3t':m3bto3t, u'name_map':nm}
from calibre.utils.serialize import msgpack_dumps
with open(dest, 'wb') as f:
f.write(msgpack_dumps(x))
def env(self):
if ispy3:
env = os.environ.copy()
else:
# We use this inefficient method of copying the environment variables
# because of non ascii env vars on windows. See https://bugs.launchpad.net/bugs/811191
env = {}
for key in os.environ:
try:
val = os.environ[key]
if isinstance(val, unicode_type):
# On windows subprocess cannot handle unicode env vars
try:
val = val.encode(filesystem_encoding)
except ValueError:
val = val.encode('utf-8')
if isinstance(key, unicode_type):
key = key.encode('ascii')
env[key] = val
except:
pass
env[native_string_type('CALIBRE_WORKER')] = environ_item('1')
td = as_hex_unicode(msgpack_dumps(base_dir()))
env[native_string_type('CALIBRE_WORKER_TEMP_DIR')] = environ_item(td)
env.update(self._env)
return env
def __call__(self, names, args, in_process_container):
num_workers = len(self.workers) + 1
if num_workers == 1:
return [process_book_files(names, *args, container=in_process_container)]
group_sz = int(ceil(len(names) / num_workers))
groups = tuple(grouper(group_sz, names))
for group, worker in zip(groups[:-1], self.workers):
worker.stdin.write(as_bytes(msgpack_dumps((worker.output_path, group,) + args)))
worker.stdin.flush(), worker.stdin.close()
worker.job_sent = True
for worker in self.workers:
if not hasattr(worker, 'job_sent'):
worker.stdin.write(b'_'), worker.stdin.flush(), worker.stdin.close()
error = None
results = [process_book_files(groups[-1], *args, container=in_process_container)]
for worker in self.workers:
if not hasattr(worker, 'job_sent'):
worker.wait()
continue
if worker.wait() != 0:
with lopen(worker.error_path, 'rb') as f:
error = f.read().decode('utf-8', 'replace')
else:
with lopen(worker.output_path, 'rb') as f:
results.append(msgpack_loads(f.read()))
if error is not None:
raise Exception('Render worker failed with error:\n' + error)
return results
def compile_main_translations(self):
l = {}
lc_dataf = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'lc_data.py')
exec(compile(open(lc_dataf, 'rb').read(), lc_dataf, 'exec'), l, l)
lcdata = {k:{k1:v1 for k1, v1 in v} for k, v in l['data']}
self.iso639_errors = []
self.info('Compiling main UI translation files...')
fmap = {f:self.mo_file(f) for f in self.po_files()}
files = [(f, fmap[f][1]) for f in self.po_files()]
def action_per_file(f):
locale, dest = fmap[f]
ln = normalize_locale(locale).partition('.')[0]
if ln in lcdata:
ld = lcdata[ln]
lcdest = self.j(self.d(dest), 'lcdata.calibre_msgpack')
from calibre.utils.serialize import msgpack_dumps
with open(lcdest, 'wb') as lcf:
lcf.write(msgpack_dumps(ld))
stats = {}
def handle_stats(f, nums):
trans = nums[0]
total = trans if len(nums) == 1 else (trans + nums[1])
locale = fmap[f][0]
stats[locale] = min(1.0, float(trans)/total)
self.compile_group(files, handle_stats=handle_stats, action_per_file=action_per_file)
self.info('Compiling ISO639 files...')
files = []
skip_iso = {
'en_GB', 'en_CA', 'en_AU', 'si', 'ur', 'sc', 'ltg', 'nds',
'te', 'yi', 'fo', 'sq', 'ast', 'ml', 'ku', 'fr_CA', 'him',
'jv', 'ka', 'fur', 'ber', 'my', 'fil', 'hy', 'ug'}
for f, (locale, dest) in iteritems(fmap):
iscpo = {'bn':'bn_IN', 'zh_HK':'zh_CN'}.get(locale, locale)
iso639 = self.j(self.TRANSLATIONS, 'iso_639', '%s.po'%iscpo)
if os.path.exists(iso639):
files.append((iso639, self.j(self.d(dest), 'iso639.mo')))
elif locale not in skip_iso:
self.warn('No ISO 639 translations for locale:', locale)
self.compile_group(files, file_ok=self.check_iso639)
if self.iso639_errors:
for err in self.iso639_errors:
print (err)
raise SystemExit(1)
dest = self.stats
base = self.d(dest)
try:
os.mkdir(base)
except EnvironmentError as err:
if err.errno != errno.EEXIST:
raise
from calibre.utils.serialize import msgpack_dumps
with open(dest, 'wb') as f:
f.write(msgpack_dumps(stats))
def run(self, opts):
src = self.j(self.d(self.SRC), 'setup', 'iso_3166-1.json')
if not os.path.exists(src):
raise Exception(src + ' does not exist')
dest = self.DEST
base = self.d(dest)
if not os.path.exists(base):
os.makedirs(base)
if not self.newer(dest, [src, __file__]):
self.info('Packed code is up to date')
return
self.info('Packing ISO-3166 codes to', dest)
with open(src, 'rb') as f:
db = json.load(f)
codes = set()
three_map = {}
name_map = {}
unicode_type = type(u'')
for x in db['3166-1']:
two = x.get('alpha_2')
if two:
two = unicode_type(two)
codes.add(two)
name_map[two] = x.get('name')
if name_map[two]:
name_map[two] = unicode_type(name_map[two])
three = x.get('alpha_3')
if three:
three_map[unicode_type(three)] = two
x = {u'names':name_map, u'codes':frozenset(codes), u'three_map':three_map}
from calibre.utils.serialize import msgpack_dumps
with open(dest, 'wb') as f:
f.write(msgpack_dumps(x))
def launch_worker(self, gui=False, redirect_output=None, job_name=None):
start = time.time()
with self._worker_launch_lock:
self.launched_worker_count += 1
id = self.launched_worker_count
fd, rfile = tempfile.mkstemp(prefix=u'ipc_result_%d_%d_' %
(self.id, id),
dir=base_dir(),
suffix=u'.pickle')
os.close(fd)
if redirect_output is None:
redirect_output = not gui
env = {
'CALIBRE_WORKER_ADDRESS':
environ_item(hexlify(msgpack_dumps(self.listener.address))),
'CALIBRE_WORKER_KEY':
environ_item(hexlify(self.auth_key)),
'CALIBRE_WORKER_RESULT':
environ_item(hexlify(rfile.encode('utf-8'))),
}
cw = self.do_launch(env,
gui,
redirect_output,
rfile,
job_name=job_name)
if isinstance(cw, string_or_bytes):
raise CriticalError('Failed to launch worker process:\n' + cw)
if DEBUG:
print('Worker Launch took:', time.time() - start)
return cw
def do_convert(path, temp_path, key, instance):
tdir = os.path.join(temp_path, instance['path'])
p = None
try:
with TemporaryFile('log.txt') as logpath:
with open(logpath, 'w+b') as logf:
p = start_pipe_worker(
'from calibre.srv.render_book import viewer_main; viewer_main()',
stdout=logf,
stderr=logf)
running_workers.append(p)
p.stdin.write(
msgpack_dumps((
path,
tdir,
{
'size': instance['file_size'],
'mtime': instance['file_mtime'],
'hash': key
},
)))
p.stdin.close()
if p.wait() != 0:
with lopen(logpath, 'rb') as logf:
worker_output = logf.read().decode('utf-8', 'replace')
raise ConversionFailure(path, worker_output)
finally:
try:
running_workers.remove(p)
except Exception:
pass
size = 0
for f in walk(tdir):
size += os.path.getsize(f)
instance['cache_size'] = size
def compile_main_translations(self):
l = {}
lc_dataf = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'lc_data.py')
exec(compile(open(lc_dataf, 'rb').read(), lc_dataf, 'exec'), l, l)
lcdata = {k:{k1:v1 for k1, v1 in v} for k, v in l['data']}
self.iso639_errors = []
self.info('Compiling main UI translation files...')
fmap = {f:self.mo_file(f) for f in self.po_files()}
files = [(f, fmap[f][1]) for f in self.po_files()]
def action_per_file(f):
locale, dest = fmap[f]
ln = normalize_locale(locale).partition('.')[0]
if ln in lcdata:
ld = lcdata[ln]
lcdest = self.j(self.d(dest), 'lcdata.calibre_msgpack')
from calibre.utils.serialize import msgpack_dumps
with open(lcdest, 'wb') as lcf:
lcf.write(msgpack_dumps(ld))
stats = {}
def handle_stats(f, nums):
trans = nums[0]
total = trans if len(nums) == 1 else (trans + nums[1])
locale = fmap[f][0]
stats[locale] = min(1.0, float(trans)/total)
self.compile_group(files, handle_stats=handle_stats, action_per_file=action_per_file)
self.info('Compiling ISO639 files...')
files = []
skip_iso = {
'en_GB', 'en_CA', 'en_AU', 'si', 'ur', 'sc', 'ltg', 'nds',
'te', 'yi', 'fo', 'sq', 'ast', 'ml', 'ku', 'fr_CA', 'him',
'jv', 'ka', 'fur', 'ber', 'my', 'fil', 'hy', 'ug'}
for f, (locale, dest) in iteritems(fmap):
iscpo = {'bn':'bn_IN', 'zh_HK':'zh_CN'}.get(locale, locale)
iso639 = self.j(self.TRANSLATIONS, 'iso_639', '%s.po'%iscpo)
if os.path.exists(iso639):
files.append((iso639, self.j(self.d(dest), 'iso639.mo')))
elif locale not in skip_iso:
self.warn('No ISO 639 translations for locale:', locale)
self.compile_group(files, file_ok=self.check_iso639)
if self.iso639_errors:
for err in self.iso639_errors:
print (err)
raise SystemExit(1)
dest = self.stats
base = self.d(dest)
try:
os.mkdir(base)
except EnvironmentError as err:
if err.errno != errno.EEXIST:
raise
from calibre.utils.serialize import msgpack_dumps
with open(dest, 'wb') as f:
f.write(msgpack_dumps(stats))
def env(self):
if ispy3:
env = os.environ.copy()
else:
# We use this inefficient method of copying the environment variables
# because of non ascii env vars on windows. See https://bugs.launchpad.net/bugs/811191
env = {}
for key in os.environ:
try:
val = os.environ[key]
if isinstance(val, unicode_type):
# On windows subprocess cannot handle unicode env vars
try:
val = val.encode(filesystem_encoding)
except ValueError:
val = val.encode('utf-8')
if isinstance(key, unicode_type):
key = key.encode('ascii')
env[key] = val
except:
pass
env[native_string_type('CALIBRE_WORKER')] = environ_item('1')
td = as_hex_unicode(msgpack_dumps(base_dir()))
env[native_string_type('CALIBRE_WORKER_TEMP_DIR')] = environ_item(td)
env.update(self._env)
return env
def launch_worker(self, gui=False, redirect_output=None, job_name=None):
start = time.monotonic()
id = next(self.launched_worker_counter)
fd, rfile = tempfile.mkstemp(prefix='ipc_result_%d_%d_' %
(self.id, id),
dir=base_dir(),
suffix='.pickle')
os.close(fd)
if redirect_output is None:
redirect_output = not gui
env = {
'CALIBRE_WORKER_ADDRESS':
environ_item(as_hex_unicode(msgpack_dumps(self.address))),
'CALIBRE_WORKER_KEY':
environ_item(as_hex_unicode(self.auth_key)),
'CALIBRE_WORKER_RESULT':
environ_item(as_hex_unicode(rfile)),
}
cw = self.do_launch(env,
gui,
redirect_output,
rfile,
job_name=job_name)
if isinstance(cw, string_or_bytes):
raise CriticalError('Failed to launch worker process:\n' +
force_unicode(cw))
if DEBUG:
print(
'Worker Launch took: {:.2f} seconds'.format(time.monotonic() -
start))
return cw
def msgpack(ctx, rd, endpoint, output):
rd.outheaders.set('Content-Type', MSGPACK_MIME, replace_all=True)
if isinstance(output, bytes) or hasattr(output, 'fileno'):
ans = output # Assume output is already msgpack encoded
else:
ans = msgpack_dumps(output)
return ans
def run(self, opts):
src = self.j(self.d(self.SRC), 'setup', 'iso3166.xml')
if not os.path.exists(src):
raise Exception(src + ' does not exist')
dest = self.DEST
base = self.d(dest)
if not os.path.exists(base):
os.makedirs(base)
if not self.newer(dest, [src, __file__]):
self.info('Packed code is up to date')
return
self.info('Packing ISO-3166 codes to', dest)
from lxml import etree
root = etree.fromstring(open(src, 'rb').read())
codes = set()
three_map = {}
name_map = {}
for x in root.xpath('//iso_3166_entry'):
two = x.get('alpha_2_code')
three = x.get('alpha_3_code')
codes.add(two)
name_map[two] = x.get('name')
if three:
three_map[three] = two
x = {
'names': name_map,
'codes': frozenset(codes),
'three_map': three_map
}
from calibre.utils.serialize import msgpack_dumps
with open(dest, 'wb') as f:
f.write(msgpack_dumps(x))
def run(self, opts):
src = self.j(self.d(self.SRC), 'setup', 'iso_639_3.xml')
if not os.path.exists(src):
raise Exception(src + ' does not exist')
dest = self.DEST
base = self.d(dest)
if not os.path.exists(base):
os.makedirs(base)
if not self.newer(dest, [src, __file__]):
self.info('Packed code is up to date')
return
self.info('Packing ISO-639 codes to', dest)
from lxml import etree
root = etree.fromstring(open(src, 'rb').read())
by_2 = {}
by_3b = {}
by_3t = {}
m2to3 = {}
m3to2 = {}
m3bto3t = {}
nm = {}
codes2, codes3t, codes3b = set(), set(), set()
for x in root.xpath('//iso_639_3_entry'):
two = x.get('part1_code', None)
threet = x.get('id')
threeb = x.get('part2_code', None)
if threeb is None:
# Only recognize languages in ISO-639-2
continue
name = x.get('name')
if two is not None:
by_2[two] = name
codes2.add(two)
m2to3[two] = threet
m3to2[threeb] = m3to2[threet] = two
by_3b[threeb] = name
by_3t[threet] = name
if threeb != threet:
m3bto3t[threeb] = threet
codes3b.add(threeb)
codes3t.add(threet)
base_name = name.lower()
nm[base_name] = threet
x = {
'by_2': by_2,
'by_3b': by_3b,
'by_3t': by_3t,
'codes2': codes2,
'codes3b': codes3b,
'codes3t': codes3t,
'2to3': m2to3,
'3to2': m3to2,
'3bto3t': m3bto3t,
'name_map': nm
}
from calibre.utils.serialize import msgpack_dumps
with open(dest, 'wb') as f:
f.write(msgpack_dumps(x))
def env(self):
env = os.environ.copy()
env[native_string_type('CALIBRE_WORKER')] = environ_item('1')
td = as_hex_unicode(msgpack_dumps(base_dir()))
env[native_string_type('CALIBRE_WORKER_TEMP_DIR')] = environ_item(td)
env.update(self._env)
return env
def run(self, opts):
src = self.j(self.d(self.SRC), 'setup', 'iso3166.xml')
if not os.path.exists(src):
raise Exception(src + ' does not exist')
dest = self.DEST
base = self.d(dest)
if not os.path.exists(base):
os.makedirs(base)
if not self.newer(dest, [src, __file__]):
self.info('Packed code is up to date')
return
self.info('Packing ISO-3166 codes to', dest)
from lxml import etree
root = etree.fromstring(open(src, 'rb').read())
codes = set()
three_map = {}
name_map = {}
for x in root.xpath('//iso_3166_entry'):
two = x.get('alpha_2_code')
three = x.get('alpha_3_code')
codes.add(two)
name_map[two] = x.get('name')
if three:
three_map[three] = two
x = {'names':name_map, 'codes':frozenset(codes), 'three_map':three_map}
from calibre.utils.serialize import msgpack_dumps
with open(dest, 'wb') as f:
f.write(msgpack_dumps(x))
def run(self, opts):
src = self.j(self.d(self.SRC), 'setup', 'iso_639-3.json')
if not os.path.exists(src):
raise Exception(src + ' does not exist')
dest = self.DEST
base = self.d(dest)
if not os.path.exists(base):
os.makedirs(base)
if not self.newer(dest, [src, __file__]):
self.info('Packed code is up to date')
return
self.info('Packing ISO-639 codes to', dest)
with open(src, 'rb') as f:
root = json.load(f)
entries = root['639-3']
by_2 = {}
by_3 = {}
m2to3 = {}
m3to2 = {}
nm = {}
codes2, codes3 = set(), set()
unicode_type = type(u'')
for x in entries:
two = x.get('alpha_2')
if two:
two = unicode_type(two)
threeb = x.get('alpha_3')
if threeb:
threeb = unicode_type(threeb)
if threeb is None:
continue
name = x.get('inverted_name') or x.get('name')
if name:
name = unicode_type(name)
if not name or name[0] in '!~=/\'"':
continue
if two is not None:
by_2[two] = name
codes2.add(two)
m2to3[two] = threeb
m3to2[threeb] = two
codes3.add(threeb)
by_3[threeb] = name
base_name = name.lower()
nm[base_name] = threeb
x = {
u'by_2': by_2,
u'by_3': by_3,
u'codes2': codes2,
u'codes3': codes3,
u'2to3': m2to3,
u'3to2': m3to2,
u'name_map': nm
}
from calibre.utils.serialize import msgpack_dumps
with open(dest, 'wb') as f:
f.write(msgpack_dumps(x))
def __call__(self, redirect_output=True, cwd=None, priority=None):
'''
If redirect_output is True, output from the child is redirected
to a file on disk and this method returns the path to that file.
'''
exe = self.gui_executable if self.gui else self.executable
env = self.env
try:
origwd = cwd or os.path.abspath(getcwd())
except EnvironmentError:
# cwd no longer exists
origwd = cwd or os.path.expanduser('~')
env[native_string_type('ORIGWD')] = environ_item(
as_hex_unicode(msgpack_dumps(origwd)))
_cwd = cwd
if priority is None:
priority = prefs['worker_process_priority']
cmd = [exe] if isinstance(exe, string_or_bytes) else exe
args = {
'env': env,
'cwd': _cwd,
}
if iswindows:
priority = {
'high': subprocess.HIGH_PRIORITY_CLASS,
'normal': subprocess.NORMAL_PRIORITY_CLASS,
'low': subprocess.IDLE_PRIORITY_CLASS
}[priority]
args['creationflags'] = subprocess.CREATE_NO_WINDOW | priority
else:
niceness = {
'normal': 0,
'low': 10,
'high': 20,
}[priority]
args['env']['CALIBRE_WORKER_NICENESS'] = str(niceness)
ret = None
if redirect_output:
self._file = PersistentTemporaryFile('_worker_redirect.log')
args['stdout'] = self._file._fd
args['stderr'] = subprocess.STDOUT
if iswindows:
args['stdin'] = subprocess.PIPE
ret = self._file.name
if iswindows and 'stdin' not in args:
# On windows when using the pythonw interpreter,
# stdout, stderr and stdin may not be valid
args['stdin'] = subprocess.PIPE
args['stdout'] = windows_null_file
args['stderr'] = subprocess.STDOUT
self.child = subprocess.Popen(cmd, **args)
if 'stdin' in args:
self.child.stdin.close()
self.log_path = ret
return ret
def action_per_file(f):
locale, dest = fmap[f]
ln = normalize_locale(locale).partition('.')[0]
if ln in lcdata:
ld = lcdata[ln]
lcdest = self.j(self.d(dest), 'lcdata.calibre_msgpack')
from calibre.utils.serialize import msgpack_dumps
with open(lcdest, 'wb') as lcf:
lcf.write(msgpack_dumps(ld))
def action_per_file(f):
locale, dest = fmap[f]
ln = normalize_locale(locale).partition('.')[0]
if ln in lcdata:
ld = lcdata[ln]
lcdest = self.j(self.d(dest), 'lcdata.calibre_msgpack')
from calibre.utils.serialize import msgpack_dumps
with open(lcdest, 'wb') as lcf:
lcf.write(msgpack_dumps(ld))
def worker_main():
stdin = getattr(sys.stdin, 'buffer', sys.stdin)
raw = stdin.read()
if raw == b'_':
return
args = msgpack_loads(raw)
result = process_book_files(*args[1:])
with open(args[0], 'wb') as f:
f.write(as_bytes(msgpack_dumps(result)))
def mkitaiji(self, src, dst):
dic = {}
for line in open(src, "r"):
line = line.decode("utf-8").strip()
if line.startswith(';;'): # skip comment
continue
if re.match(r"^$",line):
continue
pair = re.sub(r'\\u([0-9a-fA-F]{4})', lambda x:unichr(int(x.group(1),16)), line)
dic[pair[0]] = pair[1]
from calibre.utils.serialize import msgpack_dumps
with open(dst, 'wb') as f:
f.write(msgpack_dumps(dic))
def mkkanadict(self, src, dst):
dic = {}
for line in open(src, "rb"):
line = line.decode('utf-8').strip()
if line.startswith(';;'): # skip comment
continue
if re.match(r"^$", line):
continue
(alpha, kana) = line.split(' ')
dic[kana] = alpha
from calibre.utils.serialize import msgpack_dumps
with open(dst, 'wb') as f:
f.write(msgpack_dumps(dic))
def mkkanadict(self, src, dst):
dic = {}
for line in open(src, "r"):
line = line.decode("utf-8").strip()
if line.startswith(';;'): # skip comment
continue
if re.match(r"^$",line):
continue
(alpha, kana) = line.split(' ')
dic[kana] = alpha
from calibre.utils.serialize import msgpack_dumps
with open(dst, 'wb') as f:
f.write(msgpack_dumps(dic))
def mkitaiji(self, src, dst):
dic = {}
for line in open(src, "rb"):
line = line.decode('utf-8').strip()
if line.startswith(';;'): # skip comment
continue
if re.match(r"^$",line):
continue
pair = re.sub(r'\\u([0-9a-fA-F]{4})', lambda x:codepoint_to_chr(int(x.group(1),16)), line)
dic[pair[0]] = pair[1]
from calibre.utils.serialize import msgpack_dumps
with open(dst, 'wb') as f:
f.write(msgpack_dumps(dic))
def update_found(self,
calibre_version,
number_of_plugin_updates,
force=False,
no_show_popup=False):
self.last_newest_calibre_version = calibre_version
has_calibre_update = calibre_version != NO_CALIBRE_UPDATE and calibre_version[
0] > 0
has_plugin_updates = number_of_plugin_updates > 0
self.plugin_update_found(number_of_plugin_updates)
version_url = as_hex_unicode(
msgpack_dumps((calibre_version, number_of_plugin_updates)))
calibre_version = '.'.join(map(str, calibre_version))
if not has_calibre_update and not has_plugin_updates:
self.status_bar.update_label.setVisible(False)
return
if has_calibre_update:
plt = ''
if has_plugin_updates:
plt = ngettext(
' and one plugin update', ' and {} plugin updates',
number_of_plugin_updates).format(number_of_plugin_updates)
msg = ('<span style="color:green; font-weight: bold">%s: '
'<a href="update:%s">%s%s</a></span>') % (
_('Update found'), version_url, calibre_version, plt)
else:
plt = ngettext('plugin update available',
'plugin updates available',
number_of_plugin_updates)
msg = ('<a href="update:%s">%d %s</a>') % (
version_url, number_of_plugin_updates, plt)
self.status_bar.update_label.setText(msg)
self.status_bar.update_label.setVisible(True)
if has_calibre_update:
if (force or (config.get('new_version_notification')
and not is_version_notified(calibre_version))):
if not no_show_popup:
self._update_notification__ = UpdateNotification(
calibre_version, number_of_plugin_updates, parent=self)
self._update_notification__.show()
elif has_plugin_updates:
if force:
from calibre.gui2.dialogs.plugin_updater import (
PluginUpdaterDialog, FILTER_UPDATE_AVAILABLE)
d = PluginUpdaterDialog(self,
initial_filter=FILTER_UPDATE_AVAILABLE)
d.exec()
if d.do_restart:
self.quit(restart=True)
def run(self):
from calibre.utils.ipc.server import create_listener
self.auth_key = os.urandom(32)
self.address, self.listener = create_listener(self.auth_key)
self.worker_data = msgpack_dumps((self.address, self.auth_key))
if self.start_worker() is False:
return
while True:
event = self.events.get()
if event is None or self.shutting_down:
break
if self.handle_event(event) is False:
break
def test_serialize_metadata(self): # {{{
from calibre.utils.serialize import json_dumps, json_loads, msgpack_dumps, msgpack_loads
from calibre.library.field_metadata import fm_as_dict
cache = self.init_cache(self.library_path)
fm = cache.field_metadata
for d, l in ((json_dumps, json_loads), (msgpack_dumps, msgpack_loads)):
fm2 = l(d(fm))
self.assertEqual(fm_as_dict(fm), fm_as_dict(fm2))
for i in range(1, 4):
mi = cache.get_metadata(i, get_cover=True, cover_as_data=True)
rmi = msgpack_loads(msgpack_dumps(mi))
self.compare_metadata(mi, rmi, exclude='format_metadata has_cover formats id'.split())
rmi = json_loads(json_dumps(mi))
self.compare_metadata(mi, rmi, exclude='format_metadata has_cover formats id'.split())
def run(self):
from calibre.utils.ipc.server import create_listener
self.auth_key = os.urandom(32)
self.address, self.listener = create_listener(self.auth_key)
self.worker_data = msgpack_dumps((self.address, self.auth_key))
if self.start_worker() is False:
return
while True:
event = self.events.get()
if event is None or self.shutting_down:
break
if self.handle_event(event) is False:
break
def test_serialize_metadata(self): # {{{
from calibre.utils.serialize import json_dumps, json_loads, msgpack_dumps, msgpack_loads
from calibre.library.field_metadata import fm_as_dict
cache = self.init_cache(self.library_path)
fm = cache.field_metadata
for d, l in ((json_dumps, json_loads), (msgpack_dumps, msgpack_loads)):
fm2 = l(d(fm))
self.assertEqual(fm_as_dict(fm), fm_as_dict(fm2))
for i in range(1, 4):
mi = cache.get_metadata(i, get_cover=True, cover_as_data=True)
rmi = msgpack_loads(msgpack_dumps(mi))
self.compare_metadata(mi, rmi, exclude='format_metadata has_cover formats id'.split())
rmi = json_loads(json_dumps(mi))
self.compare_metadata(mi, rmi, exclude='format_metadata has_cover formats id'.split())
def launch_worker_process(self):
from calibre.utils.ipc.server import create_listener
from calibre.utils.ipc.pool import start_worker
self.worker_process = p = start_worker(
'from {0} import run_main, {1}; run_main({1})'.format(self.__class__.__module__, self.worker_entry_point))
auth_key = os.urandom(32)
address, self.listener = create_listener(auth_key)
eintr_retry_call(p.stdin.write, msgpack_dumps((address, auth_key)))
p.stdin.flush(), p.stdin.close()
self.control_conn = eintr_retry_call(self.listener.accept)
self.data_conn = eintr_retry_call(self.listener.accept)
self.data_thread = t = Thread(name='CWData', target=self.handle_data_requests)
t.daemon = True
t.start()
self.connected.set()
def create_worker(env, priority='normal', cwd=None, func='main'):
from calibre.utils.ipc.server import create_listener
auth_key = os.urandom(32)
address, listener = create_listener(auth_key)
env = dict(env)
env.update({
'CALIBRE_WORKER_ADDRESS': environ_item(as_hex_unicode(msgpack_dumps(address))),
'CALIBRE_WORKER_KEY': environ_item(as_hex_unicode(auth_key)),
'CALIBRE_SIMPLE_WORKER': environ_item('calibre.utils.ipc.simple_worker:%s' % func),
})
w = Worker(env)
w(cwd=cwd, priority=priority)
return listener, w
def create_worker(env, priority='normal', cwd=None, func='main'):
from calibre.utils.ipc.server import create_listener
auth_key = os.urandom(32)
address, listener = create_listener(auth_key)
env = dict(env)
env.update({
'CALIBRE_WORKER_ADDRESS': environ_item(as_hex_unicode(msgpack_dumps(address))),
'CALIBRE_WORKER_KEY': environ_item(as_hex_unicode(auth_key)),
'CALIBRE_SIMPLE_WORKER': environ_item('calibre.utils.ipc.simple_worker:%s' % func),
})
w = Worker(env)
w(cwd=cwd, priority=priority)
return listener, w
def launch_worker_process(self):
from calibre.utils.ipc.server import create_listener
from calibre.utils.ipc.pool import start_worker
self.worker_process = p = start_worker(
'from {0} import run_main, {1}; run_main({1})'.format(self.__class__.__module__, self.worker_entry_point))
auth_key = os.urandom(32)
address, self.listener = create_listener(auth_key)
eintr_retry_call(p.stdin.write, msgpack_dumps((address, auth_key)))
p.stdin.flush(), p.stdin.close()
self.control_conn = eintr_retry_call(self.listener.accept)
self.data_conn = eintr_retry_call(self.listener.accept)
self.data_thread = t = Thread(name='CWData', target=self.handle_data_requests)
t.daemon = True
t.start()
self.connected.set()
def run(self):
try:
with PersistentTemporaryFile('print-to-pdf-log.txt') as f:
p = self.worker = start_pipe_worker('from calibre.gui2.viewer.printing import do_print; do_print()', stdout=f, stderr=subprocess.STDOUT)
p.stdin.write(msgpack_dumps(self.data)), p.stdin.flush(), p.stdin.close()
rc = p.wait()
if rc != 0:
f.seek(0)
self.log = f.read().decode('utf-8', 'replace')
try:
os.remove(f.name)
except EnvironmentError:
pass
except Exception:
import traceback
self.tb = traceback.format_exc()
def run(self):
try:
with PersistentTemporaryFile('print-to-pdf-log.txt') as f:
p = self.worker = start_pipe_worker('from calibre.gui2.viewer.printing import do_print; do_print()', stdout=f, stderr=subprocess.STDOUT)
p.stdin.write(msgpack_dumps(self.data)), p.stdin.flush(), p.stdin.close()
rc = p.wait()
if rc != 0:
f.seek(0)
self.log = f.read().decode('utf-8', 'replace')
try:
os.remove(f.name)
except EnvironmentError:
pass
except Exception:
import traceback
self.tb = traceback.format_exc()
def remote_run(self, name, m, *args):
from mechanize import HTTPError, Request
from calibre.utils.serialize import msgpack_loads, msgpack_dumps
url = self.url + '/cdb/cmd/{}/{}'.format(name, getattr(m, 'version', 0))
if self.library_id:
url += '?' + urlencode({'library_id':self.library_id})
rq = Request(url, data=msgpack_dumps(args),
headers={'Accept': MSGPACK_MIME, 'Content-Type': MSGPACK_MIME})
try:
res = self.br.open_novisit(rq)
ans = msgpack_loads(res.read())
except HTTPError as err:
self.interpret_http_error(err)
raise
if 'err' in ans:
prints(ans['tb'])
raise SystemExit(ans['err'])
return ans['result']
def remote_run(self, name, m, *args):
from mechanize import HTTPError, Request
from calibre.utils.serialize import msgpack_loads, msgpack_dumps
url = self.url + '/cdb/cmd/{}/{}'.format(name, getattr(m, 'version', 0))
if self.library_id:
url += '?' + urlencode({'library_id':self.library_id})
rq = Request(url, data=msgpack_dumps(args),
headers={'Accept': MSGPACK_MIME, 'Content-Type': MSGPACK_MIME})
try:
res = self.br.open_novisit(rq, timeout=self.timeout)
ans = msgpack_loads(res.read())
except HTTPError as err:
self.interpret_http_error(err)
raise
if 'err' in ans:
if ans['tb']:
prints(ans['tb'])
raise SystemExit(ans['err'])
return ans['result']
def launch_worker(self, gui=False, redirect_output=None, job_name=None):
start = time.time()
with self._worker_launch_lock:
self.launched_worker_count += 1
id = self.launched_worker_count
fd, rfile = tempfile.mkstemp(prefix=u'ipc_result_%d_%d_'%(self.id, id),
dir=base_dir(), suffix=u'.pickle')
os.close(fd)
if redirect_output is None:
redirect_output = not gui
env = {
'CALIBRE_WORKER_ADDRESS' : environ_item(as_hex_unicode(msgpack_dumps(self.address))),
'CALIBRE_WORKER_KEY' : environ_item(as_hex_unicode(self.auth_key)),
'CALIBRE_WORKER_RESULT' : environ_item(as_hex_unicode(rfile)),
}
cw = self.do_launch(env, gui, redirect_output, rfile, job_name=job_name)
if isinstance(cw, string_or_bytes):
raise CriticalError('Failed to launch worker process:\n'+cw)
if DEBUG:
print('Worker Launch took:', time.time() - start)
return cw
def run(self, opts):
from calibre.utils.serialize import msgpack_dumps
scripts = {}
for x in ('console', 'gui'):
for name in basenames[x]:
if name in ('calibre-complete', 'calibre_postinstall'):
continue
scripts[name] = x
dest = self.j(self.RESOURCES, 'scripts.calibre_msgpack')
if self.newer(dest, self.j(self.SRC, 'calibre', 'linux.py')):
self.info('\tCreating ' + os.path.basename(dest))
with open(dest, 'wb') as f:
f.write(msgpack_dumps(scripts))
from calibre.web.feeds.recipes.collection import \
serialize_builtin_recipes, iterate_over_builtin_recipe_files
files = [x[1] for x in iterate_over_builtin_recipe_files()]
dest = self.j(self.RESOURCES, 'builtin_recipes.xml')
if self.newer(dest, files):
self.info('\tCreating builtin_recipes.xml')
xml = serialize_builtin_recipes()
with open(dest, 'wb') as f:
f.write(xml)
recipe_icon_dir = self.a(self.j(self.RESOURCES, '..', 'recipes',
'icons'))
dest = os.path.splitext(dest)[0] + '.zip'
files += glob.glob(self.j(recipe_icon_dir, '*.png'))
if self.newer(dest, files):
self.info('\tCreating builtin_recipes.zip')
with zipfile.ZipFile(dest, 'w', zipfile.ZIP_STORED) as zf:
for n in sorted(files, key=self.b):
with open(n, 'rb') as f:
zf.writestr(os.path.basename(n), f.read())
dest = self.j(self.RESOURCES, 'ebook-convert-complete.calibre_msgpack')
files = []
for x in os.walk(self.j(self.SRC, 'calibre')):
for f in x[-1]:
if f.endswith('.py'):
files.append(self.j(x[0], f))
if self.newer(dest, files):
self.info('\tCreating ebook-convert-complete.pickle')
complete = {}
from calibre.ebooks.conversion.plumber import supported_input_formats
complete['input_fmts'] = set(supported_input_formats())
from calibre.web.feeds.recipes.collection import get_builtin_recipe_titles
complete['input_recipes'] = [t+'.recipe ' for t in
get_builtin_recipe_titles()]
from calibre.customize.ui import available_output_formats
complete['output'] = set(available_output_formats())
from calibre.ebooks.conversion.cli import create_option_parser
from calibre.utils.logging import Log
log = Log()
# log.outputs = []
for inf in supported_input_formats():
if inf in ('zip', 'rar', 'oebzip'):
continue
for ouf in available_output_formats():
of = ouf if ouf == 'oeb' else 'dummy.'+ouf
p = create_option_parser(('ec', 'dummy1.'+inf, of, '-h'),
log)[0]
complete[(inf, ouf)] = [x+' 'for x in
get_opts_from_parser(p)]
with open(dest, 'wb') as f:
f.write(msgpack_dumps(complete))
self.info('\tCreating template-functions.json')
dest = self.j(self.RESOURCES, 'template-functions.json')
function_dict = {}
import inspect
from calibre.utils.formatter_functions import formatter_functions
for obj in formatter_functions().get_builtins().values():
eval_func = inspect.getmembers(obj,
lambda x: inspect.ismethod(x) and x.__name__ == 'evaluate')
try:
lines = [l[4:] for l in inspect.getsourcelines(eval_func[0][1])[0]]
except:
continue
lines = ''.join(lines)
function_dict[obj.name] = lines
import json
json.dump(function_dict, open(dest, 'wb'), indent=4)
self.info('\tCreating editor-functions.json')
dest = self.j(self.RESOURCES, 'editor-functions.json')
function_dict = {}
from calibre.gui2.tweak_book.function_replace import builtin_functions
for func in builtin_functions():
try:
src = ''.join(inspect.getsourcelines(func)[0][1:])
except Exception:
continue
src = src.replace('def ' + func.func_name, 'def replace')
imports = ['from %s import %s' % (x.__module__, x.__name__) for x in func.imports]
if imports:
src = '\n'.join(imports) + '\n\n' + src
function_dict[func.name] = src
json.dump(function_dict, open(dest, 'wb'), indent=4)
self.info('\tCreating user-manual-translation-stats.json')
d = {}
for lc, stats in iteritems(json.load(open(self.j(self.d(self.SRC), 'manual', 'locale', 'completed.json')))):
total = sum(itervalues(stats))
d[lc] = stats['translated'] / float(total)
json.dump(d, open(self.j(self.RESOURCES, 'user-manual-translation-stats.json'), 'wb'), indent=4)
def __call__(self, redirect_output=True, cwd=None, priority=None):
'''
If redirect_output is True, output from the child is redirected
to a file on disk and this method returns the path to that file.
'''
exe = self.gui_executable if self.gui else self.executable
env = self.env
try:
origwd = cwd or os.path.abspath(os.getcwdu())
except EnvironmentError:
# cwd no longer exists
origwd = cwd or os.path.expanduser(u'~')
env[native_string_type('ORIGWD')] = environ_item(as_hex_unicode(msgpack_dumps(origwd)))
_cwd = cwd
if priority is None:
priority = prefs['worker_process_priority']
cmd = [exe] if isinstance(exe, string_or_bytes) else exe
args = {
'env' : env,
'cwd' : _cwd,
}
if iswindows:
priority = {
'high' : win32process.HIGH_PRIORITY_CLASS,
'normal' : win32process.NORMAL_PRIORITY_CLASS,
'low' : win32process.IDLE_PRIORITY_CLASS}[priority]
args['creationflags'] = win32process.CREATE_NO_WINDOW|priority
else:
niceness = {
'normal' : 0,
'low' : 10,
'high' : 20,
}[priority]
args['preexec_fn'] = partial(renice, niceness)
ret = None
if redirect_output:
self._file = PersistentTemporaryFile('_worker_redirect.log')
args['stdout'] = self._file._fd
args['stderr'] = subprocess.STDOUT
if iswindows:
args['stdin'] = subprocess.PIPE
ret = self._file.name
if iswindows and 'stdin' not in args:
# On windows when using the pythonw interpreter,
# stdout, stderr and stdin may not be valid
args['stdin'] = subprocess.PIPE
args['stdout'] = windows_null_file
args['stderr'] = subprocess.STDOUT
if not iswindows:
# Close inherited file descriptors in worker
# On windows, this is done in the worker process
# itself
args['close_fds'] = True
self.child = subprocess.Popen(cmd, **args)
if 'stdin' in args:
self.child.stdin.close()
self.log_path = ret
return ret
def run(self, opts):
from calibre.utils.serialize import msgpack_dumps
scripts = {}
for x in ('console', 'gui'):
for name in basenames[x]:
if name in ('calibre-complete', 'calibre_postinstall'):
continue
scripts[name] = x
dest = self.j(self.RESOURCES, 'scripts.calibre_msgpack')
if self.newer(dest, self.j(self.SRC, 'calibre', 'linux.py')):
self.info('\tCreating ' + self.b(dest))
with open(dest, 'wb') as f:
f.write(msgpack_dumps(scripts))
from calibre.web.feeds.recipes.collection import \
serialize_builtin_recipes, iterate_over_builtin_recipe_files
files = [x[1] for x in iterate_over_builtin_recipe_files()]
dest = self.j(self.RESOURCES, 'builtin_recipes.xml')
if self.newer(dest, files):
self.info('\tCreating builtin_recipes.xml')
xml = serialize_builtin_recipes()
with open(dest, 'wb') as f:
f.write(xml)
recipe_icon_dir = self.a(
self.j(self.RESOURCES, '..', 'recipes', 'icons'))
dest = os.path.splitext(dest)[0] + '.zip'
files += glob.glob(self.j(recipe_icon_dir, '*.png'))
if self.newer(dest, files):
self.info('\tCreating builtin_recipes.zip')
with zipfile.ZipFile(dest, 'w', zipfile.ZIP_STORED) as zf:
for n in sorted(files, key=self.b):
with open(n, 'rb') as f:
zf.writestr(self.b(n), f.read())
dest = self.j(self.RESOURCES, 'ebook-convert-complete.calibre_msgpack')
files = []
for x in os.walk(self.j(self.SRC, 'calibre')):
for f in x[-1]:
if f.endswith('.py'):
files.append(self.j(x[0], f))
if self.newer(dest, files):
self.info('\tCreating ' + self.b(dest))
complete = {}
from calibre.ebooks.conversion.plumber import supported_input_formats
complete['input_fmts'] = set(supported_input_formats())
from calibre.web.feeds.recipes.collection import get_builtin_recipe_titles
complete['input_recipes'] = [
t + '.recipe ' for t in get_builtin_recipe_titles()
]
from calibre.customize.ui import available_output_formats
complete['output'] = set(available_output_formats())
from calibre.ebooks.conversion.cli import create_option_parser
from calibre.utils.logging import Log
log = Log()
# log.outputs = []
for inf in supported_input_formats():
if inf in ('zip', 'rar', 'oebzip'):
continue
for ouf in available_output_formats():
of = ouf if ouf == 'oeb' else 'dummy.' + ouf
p = create_option_parser(('ec', 'dummy1.' + inf, of, '-h'),
log)[0]
complete[(inf, ouf)] = [
x + ' ' for x in get_opts_from_parser(p)
]
with open(dest, 'wb') as f:
f.write(msgpack_dumps(only_unicode_recursive(complete)))
self.info('\tCreating template-functions.json')
dest = self.j(self.RESOURCES, 'template-functions.json')
function_dict = {}
import inspect
from calibre.utils.formatter_functions import formatter_functions
for obj in formatter_functions().get_builtins().values():
eval_func = inspect.getmembers(
obj,
lambda x: inspect.ismethod(x) and x.__name__ == 'evaluate')
try:
lines = [
l[4:] for l in inspect.getsourcelines(eval_func[0][1])[0]
]
except:
continue
lines = ''.join(lines)
function_dict[obj.name] = lines
dump_json(function_dict, dest)
self.info('\tCreating editor-functions.json')
dest = self.j(self.RESOURCES, 'editor-functions.json')
function_dict = {}
from calibre.gui2.tweak_book.function_replace import builtin_functions
for func in builtin_functions():
try:
src = ''.join(inspect.getsourcelines(func)[0][1:])
except Exception:
continue
src = src.replace('def ' + func.__name__, 'def replace')
imports = [
f'from {x.__module__} import {x.__name__}'
for x in func.imports
]
if imports:
src = '\n'.join(imports) + '\n\n' + src
function_dict[func.name] = src
dump_json(function_dict, dest)
self.info('\tCreating user-manual-translation-stats.json')
d = {}
for lc, stats in iteritems(
json.load(
open(
self.j(self.d(self.SRC), 'manual', 'locale',
'completed.json')))):
total = sum(itervalues(stats))
d[lc] = stats['translated'] / float(total)
dump_json(d,
self.j(self.RESOURCES, 'user-manual-translation-stats.json'))
src = self.j(self.SRC, '..', 'Changelog.txt')
dest = self.j(self.RESOURCES, 'changelog.json')
if self.newer(dest, [src]):
self.info('\tCreating changelog.json')
from setup.changelog import parse
with open(src, encoding='utf-8') as f:
dump_json(parse(f.read(), parse_dates=False), dest)
def find_icons():
global icon_data
if icon_data is not None:
return icon_data
base_dirs = [(os.environ.get('XDG_DATA_HOME')
or os.path.expanduser('~/.local/share')) + '/icons']
base_dirs += [os.path.expanduser('~/.icons')]
base_dirs += [
os.path.join(b, 'icons') for b in os.environ.get(
'XDG_DATA_DIRS', '/usr/local/share:/usr/share').split(os.pathsep)
] + ['/usr/share/pixmaps']
ans = defaultdict(list)
sz_pat = re.compile(r'/((?:\d+x\d+)|scalable)/')
cache_file = os.path.join(cache_dir(), 'icon-theme-cache.calibre_msgpack')
exts = {'.svg', '.png', '.xpm'}
def read_icon_theme_dir(dirpath):
ans = defaultdict(list)
for path in walk(dirpath):
bn = os.path.basename(path)
name, ext = os.path.splitext(bn)
if ext in exts:
sz = sz_pat.findall(path)
if sz:
sz = sz[-1]
if sz == 'scalable':
sz = 100000
else:
sz = int(sz.partition('x')[0])
idx = len(ans[name])
ans[name].append((-sz, idx, sz, path))
for icons in itervalues(ans):
icons.sort(key=list)
return {k: (-v[0][2], v[0][3]) for k, v in iteritems(ans)}
try:
with open(cache_file, 'rb') as f:
cache = f.read()
cache = msgpack_loads(cache)
mtimes, cache = defaultdict(int, cache['mtimes']), defaultdict(
dict, cache['data'])
except Exception:
mtimes, cache = defaultdict(int), defaultdict(dict)
seen_dirs = set()
changed = False
for loc in base_dirs:
try:
subdirs = os.listdir(loc)
except EnvironmentError:
continue
for dname in subdirs:
d = os.path.join(loc, dname)
if os.path.isdir(d):
try:
mtime = os.stat(d).st_mtime
except EnvironmentError:
continue
seen_dirs.add(d)
if mtime != mtimes[d]:
changed = True
try:
cache[d] = read_icon_theme_dir(d)
except Exception:
prints(
'Failed to read icon theme dir: %r with error:' %
d)
import traceback
traceback.print_exc()
mtimes[d] = mtime
for name, data in iteritems(cache[d]):
ans[name].append(data)
for removed in set(mtimes) - seen_dirs:
mtimes.pop(removed), cache.pop(removed)
changed = True
if changed:
data = msgpack_dumps({'data': cache, 'mtimes': mtimes})
try:
with open(cache_file, 'wb') as f:
f.write(data)
except Exception:
import traceback
traceback.print_exc()
for icons in itervalues(ans):
icons.sort(key=list)
icon_data = {k: v[0][1] for k, v in iteritems(ans)}
return icon_data
