def compile_group(self,
files,
handle_stats=None,
file_ok=None,
action_per_file=None):
from calibre.constants import islinux
jobs, ok_files = [], []
hashmap = {}
def stats_cache(src, data=None):
cname = self.cache_name(src) + '.stats.json'
with open(self.j(self.cache_dir, cname),
('rb' if data is None else 'wb')) as f:
if data is None:
return json.loads(f.read())
data = json.dumps(data)
if not isinstance(data, bytes):
data = data.encode('utf-8')
f.write(data)
for src, dest in files:
base = os.path.dirname(dest)
if not os.path.exists(base):
os.makedirs(base)
data, h = self.hash_and_data(src)
current_hash = h.digest()
saved_hash, saved_data = self.read_cache(src)
if current_hash == saved_hash:
with open(dest, 'wb') as d:
d.write(saved_data)
if handle_stats is not None:
handle_stats(src, stats_cache(src))
else:
if file_ok is None or file_ok(data, src):
self.info('\t' + os.path.relpath(
src, self.j(self.d(self.SRC), 'translations')))
if islinux:
msgfmt = ['msgfmt']
else:
msgfmt = [
sys.executable,
self.j(self.SRC, 'calibre', 'translations',
'msgfmt.py')
]
jobs.append(msgfmt + ['--statistics', '-o', dest, src])
ok_files.append((src, dest))
hashmap[src] = current_hash
if action_per_file is not None:
action_per_file(src)
for (src, dest), line in zip(ok_files,
parallel_check_output(jobs, self.info)):
self.write_cache(open(dest, 'rb').read(), hashmap[src], src)
nums = tuple(map(int, re.findall(r'\d+', line)))
stats_cache(src, nums)
if handle_stats is not None:
handle_stats(src, nums)
def run(self, opts):
self.compile_content_server_translations()
l = {}
exec(
compile(
open(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
'lc_data.py')).read(),
os.path.join(os.path.dirname(os.path.abspath(__file__)),
'lc_data.py'), 'exec'), l, l)
lcdata = {k: {k1: v1 for k1, v1 in v} for k, v in l['data']}
self.iso639_errors = []
jobs = []
for f in self.po_files():
locale, dest = self.mo_file(f)
base = os.path.dirname(dest)
if not os.path.exists(base):
os.makedirs(base)
jobs.append(['msgfmt', '-o', dest, f])
iscpo = {'bn': 'bn_IN', 'zh_HK': 'zh_CN'}.get(locale, locale)
iso639 = self.j(self.TRANSLATIONS, 'iso_639', '%s.po' % iscpo)
if os.path.exists(iso639) and self.check_iso639(iso639):
dest = self.j(self.d(dest), 'iso639.mo')
if self.newer(dest, iso639):
jobs.append(['msgfmt', '-o', dest, iso639])
elif locale not in {
'en_GB', 'en_CA', 'en_AU', 'si', 'ur', 'sc', 'ltg', 'nds',
'te', 'yi', 'fo', 'sq', 'ast', 'ml', 'ku', 'fr_CA', 'him',
'jv', 'ka', 'fur', 'ber', 'my', 'fil', 'hy', 'ug'
}:
self.warn('No ISO 639 translations for locale:', locale)
ln = normalize_locale(locale).partition('.')[0]
if ln in lcdata:
ld = lcdata[ln]
lcdest = self.j(self.d(dest), 'lcdata.pickle')
with open(lcdest, 'wb') as lcf:
lcf.write(cPickle.dumps(ld, -1))
self.info('\nCompiling %d translation files...' % len(jobs))
tuple(parallel_check_output(jobs, self.info))
if self.iso639_errors:
for err in self.iso639_errors:
print(err)
raise SystemExit(1)
self.write_stats()
self.freeze_locales()
self.compile_user_manual_translations()
def write_stats(self):
files = self.po_files()
dest = self.stats
if not self.newer(dest, files):
return
self.info('Calculating translation statistics...')
stats = {}
jobs = (['msgfmt', '--statistics', '-o', os.devnull, x] for x in files)
for f, line in zip(files, parallel_check_output(jobs, self.info)):
nums = tuple(map(int, re.findall(r'\d+', line)))
trans = nums[0]
total = trans if len(nums) == 1 else (trans + nums[1])
locale = self.mo_file(f)[0]
stats[locale] = min(1.0, float(trans) / total)
cPickle.dump(stats, open(dest, 'wb'), -1)
def compile_group(self, files, handle_stats=None, file_ok=None, action_per_file=None):
from calibre.constants import islinux
jobs, ok_files = [], []
hashmap = {}
def stats_cache(src, data=None):
cname = self.cache_name(src) + '.stats.json'
with open(self.j(self.cache_dir, cname), ('rb' if data is None else 'wb')) as f:
if data is None:
return json.loads(f.read())
data = json.dumps(data)
if not isinstance(data, bytes):
data = data.encode('utf-8')
f.write(data)
for src, dest in files:
base = os.path.dirname(dest)
if not os.path.exists(base):
os.makedirs(base)
data, h = self.hash_and_data(src)
current_hash = h.digest()
saved_hash, saved_data = self.read_cache(src)
if current_hash == saved_hash:
with open(dest, 'wb') as d:
d.write(saved_data)
if handle_stats is not None:
handle_stats(src, stats_cache(src))
else:
if file_ok is None or file_ok(data, src):
self.info('\t' + os.path.relpath(src, self.j(self.d(self.SRC), 'translations')))
if islinux:
msgfmt = ['msgfmt']
else:
msgfmt = [sys.executable, self.j(self.SRC, 'calibre', 'translations', 'msgfmt.py')]
jobs.append(msgfmt + ['--statistics', '-o', dest, src])
ok_files.append((src, dest))
hashmap[src] = current_hash
if action_per_file is not None:
action_per_file(src)
for (src, dest), line in zip(ok_files, parallel_check_output(jobs, self.info)):
self.write_cache(open(dest, 'rb').read(), hashmap[src], src)
nums = tuple(map(int, re.findall(r'\d+', line)))
stats_cache(src, nums)
if handle_stats is not None:
handle_stats(src, nums)
def write_stats(self):
files = self.po_files()
dest = self.stats
if not self.newer(dest, files):
return
self.info('Calculating translation statistics...')
stats = {}
jobs = (
['msgfmt', '--statistics', '-o', os.devnull, x] for x in files
)
for f, line in zip(files, parallel_check_output(jobs, self.info)):
nums = tuple(map(int, re.findall(r'\d+', line)))
trans = nums[0]
total = trans if len(nums) == 1 else (trans + nums[1])
locale = self.mo_file(f)[0]
stats[locale] = min(1.0, float(trans)/total)
cPickle.dump(stats, open(dest, 'wb'), -1)
def run(self, opts):
self.compile_content_server_translations()
l = {}
exec(compile(open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'lc_data.py'))
.read(), os.path.join(os.path.dirname(os.path.abspath(__file__)), 'lc_data.py'), 'exec'), l, l)
lcdata = {k:{k1:v1 for k1, v1 in v} for k, v in l['data']}
self.iso639_errors = []
jobs = []
for f in self.po_files():
locale, dest = self.mo_file(f)
base = os.path.dirname(dest)
if not os.path.exists(base):
os.makedirs(base)
jobs.append(['msgfmt', '-o', dest, f])
iscpo = {'bn':'bn_IN', 'zh_HK':'zh_CN'}.get(locale, locale)
iso639 = self.j(self.TRANSLATIONS, 'iso_639', '%s.po'%iscpo)
if os.path.exists(iso639) and self.check_iso639(iso639):
dest = self.j(self.d(dest), 'iso639.mo')
if self.newer(dest, iso639):
jobs.append(['msgfmt', '-o', dest, iso639])
elif locale not in {
'en_GB', 'en_CA', 'en_AU', 'si', 'ur', 'sc', 'ltg', 'nds',
'te', 'yi', 'fo', 'sq', 'ast', 'ml', 'ku', 'fr_CA', 'him',
'jv', 'ka', 'fur', 'ber', 'my', 'fil', 'hy', 'ug'}:
self.warn('No ISO 639 translations for locale:', locale)
ln = normalize_locale(locale).partition('.')[0]
if ln in lcdata:
ld = lcdata[ln]
lcdest = self.j(self.d(dest), 'lcdata.pickle')
with open(lcdest, 'wb') as lcf:
lcf.write(cPickle.dumps(ld, -1))
self.info('\nCompiling %d translation files...' % len(jobs))
tuple(parallel_check_output(jobs, self.info))
if self.iso639_errors:
for err in self.iso639_errors:
print (err)
raise SystemExit(1)
self.write_stats()
self.freeze_locales()
self.compile_user_manual_translations()
def compile_user_manual_translations(self):
self.info('Compiling user manual translations...')
srcbase = self.j(self.d(self.SRC), 'translations', 'manual')
destbase = self.j(self.d(self.SRC), 'manual', 'locale')
complete = {}
for x in os.listdir(srcbase):
q = self.j(srcbase, x)
if not os.path.isdir(q):
continue
dest = self.j(destbase, x, 'LC_MESSAGES')
if os.path.exists(dest):
shutil.rmtree(dest)
os.makedirs(dest)
jobs = []
for po in os.listdir(q):
if not po.endswith('.po'):
continue
jobs.append([
'msgfmt', '--statistics', '-o',
self.j(dest,
po.rpartition('.')[0] + '.mo'),
self.j(q, po)
])
stats = tuple(parallel_check_output(jobs, self.info))
translated = untranslated = 0
for line in stats:
m = re.search('(\d+).+(\d+)', line)
if m is None:
translated += int(re.search('\d+', line).group())
else:
translated += int(m.group(1))
untranslated += int(m.group(2))
stats = {'translated': translated, 'untranslated': untranslated}
with open(self.j(self.d(dest), 'stats.json'), 'wb') as f:
json.dump(stats, f)
total = translated + untranslated
if total and (translated / float(total)) > 0.75:
complete[x] = stats
with open(self.j(destbase, 'completed.json'), 'wb') as f:
json.dump(complete, f)
def compile_user_manual_translations(self):
self.info('Compiling user manual translations...')
srcbase = self.j(self.d(self.SRC), 'translations', 'manual')
destbase = self.j(self.d(self.SRC), 'manual', 'locale')
complete = {}
for x in os.listdir(srcbase):
q = self.j(srcbase, x)
if not os.path.isdir(q):
continue
dest = self.j(destbase, x, 'LC_MESSAGES')
if os.path.exists(dest):
shutil.rmtree(dest)
os.makedirs(dest)
jobs = []
for po in os.listdir(q):
if not po.endswith('.po'):
continue
jobs.append([
'msgfmt', '--statistics', '-o',
self.j(dest,
po.rpartition('.')[0] + '.mo'),
self.j(q, po)
])
stats = tuple(parallel_check_output(jobs, self.info))
translated = untranslated = 0
for line in stats:
nums = tuple(map(int, re.findall(r'\d+', line)))
translated += nums[0]
if len(nums) > 1:
untranslated += nums[1]
stats = {'translated': translated, 'untranslated': untranslated}
with open(self.j(self.d(dest), 'stats.json'), 'wb') as f:
json.dump(stats, f)
total = translated + untranslated
# Raise the 30% threshold in the future
if total and (translated / float(total)) > 0.3:
complete[x] = stats
with open(self.j(destbase, 'completed.json'), 'wb') as f:
json.dump(complete, f, indent=True, sort_keys=True)
def compile_user_manual_translations(self):
self.info('Compiling user manual translations...')
srcbase = self.j(self.d(self.SRC), 'translations', 'manual')
destbase = self.j(self.d(self.SRC), 'manual', 'locale')
complete = {}
for x in os.listdir(srcbase):
q = self.j(srcbase, x)
if not os.path.isdir(q):
continue
dest = self.j(destbase, x, 'LC_MESSAGES')
if os.path.exists(dest):
shutil.rmtree(dest)
os.makedirs(dest)
jobs = []
for po in os.listdir(q):
if not po.endswith('.po'):
continue
jobs.append([
'msgfmt', '--statistics', '-o', self.j(
dest, po.rpartition('.')[0] + '.mo'), self.j(q, po)])
stats = tuple(parallel_check_output(jobs, self.info))
translated = untranslated = 0
for line in stats:
m = re.search('(\d+).+(\d+)', line)
if m is None:
translated += int(re.search('\d+', line).group())
else:
translated += int(m.group(1))
untranslated += int(m.group(2))
stats = {'translated':translated, 'untranslated':untranslated}
with open(self.j(self.d(dest), 'stats.json'), 'wb') as f:
json.dump(stats, f)
total = translated + untranslated
if total and (translated / float(total)) > 0.75:
complete[x] = stats
with open(self.j(destbase, 'completed.json'), 'wb') as f:
json.dump(complete, f)
def compile_user_manual_translations(self):
self.info('Compiling user manual translations...')
srcbase = self.j(self.d(self.SRC), 'translations', 'manual')
destbase = self.j(self.d(self.SRC), 'manual', 'locale')
complete = {}
for x in os.listdir(srcbase):
q = self.j(srcbase, x)
if not os.path.isdir(q):
continue
dest = self.j(destbase, x, 'LC_MESSAGES')
if os.path.exists(dest):
shutil.rmtree(dest)
os.makedirs(dest)
jobs = []
for po in os.listdir(q):
if not po.endswith('.po'):
continue
jobs.append([
'msgfmt', '--statistics', '-o', self.j(
dest, po.rpartition('.')[0] + '.mo'), self.j(q, po)])
stats = tuple(parallel_check_output(jobs, self.info))
translated = untranslated = 0
for line in stats:
nums = tuple(map(int, re.findall(r'\d+', line)))
translated += nums[0]
if len(nums) > 1:
untranslated += nums[1]
stats = {'translated':translated, 'untranslated':untranslated}
with open(self.j(self.d(dest), 'stats.json'), 'wb') as f:
json.dump(stats, f)
total = translated + untranslated
# Raise the 30% threshold in the future
if total and (translated / float(total)) > 0.3:
complete[x] = stats
with open(self.j(destbase, 'completed.json'), 'wb') as f:
json.dump(complete, f, indent=True, sort_keys=True)
