def compile_group(self, files, handle_stats=None, file_ok=None, action_per_file=None): from calibre.constants import islinux jobs, ok_files = [], [] hashmap = {} def stats_cache(src, data=None): cname = self.cache_name(src) + '.stats.json' with open(self.j(self.cache_dir, cname), ('rb' if data is None else 'wb')) as f: if data is None: return json.loads(f.read()) data = json.dumps(data) if not isinstance(data, bytes): data = data.encode('utf-8') f.write(data) for src, dest in files: base = os.path.dirname(dest) if not os.path.exists(base): os.makedirs(base) data, h = self.hash_and_data(src) current_hash = h.digest() saved_hash, saved_data = self.read_cache(src) if current_hash == saved_hash: with open(dest, 'wb') as d: d.write(saved_data) if handle_stats is not None: handle_stats(src, stats_cache(src)) else: if file_ok is None or file_ok(data, src): self.info('\t' + os.path.relpath( src, self.j(self.d(self.SRC), 'translations'))) if islinux: msgfmt = ['msgfmt'] else: msgfmt = [ sys.executable, self.j(self.SRC, 'calibre', 'translations', 'msgfmt.py') ] jobs.append(msgfmt + ['--statistics', '-o', dest, src]) ok_files.append((src, dest)) hashmap[src] = current_hash if action_per_file is not None: action_per_file(src) for (src, dest), line in zip(ok_files, parallel_check_output(jobs, self.info)): self.write_cache(open(dest, 'rb').read(), hashmap[src], src) nums = tuple(map(int, re.findall(r'\d+', line))) stats_cache(src, nums) if handle_stats is not None: handle_stats(src, nums)
def run(self, opts): self.compile_content_server_translations() l = {} exec( compile( open( os.path.join(os.path.dirname(os.path.abspath(__file__)), 'lc_data.py')).read(), os.path.join(os.path.dirname(os.path.abspath(__file__)), 'lc_data.py'), 'exec'), l, l) lcdata = {k: {k1: v1 for k1, v1 in v} for k, v in l['data']} self.iso639_errors = [] jobs = [] for f in self.po_files(): locale, dest = self.mo_file(f) base = os.path.dirname(dest) if not os.path.exists(base): os.makedirs(base) jobs.append(['msgfmt', '-o', dest, f]) iscpo = {'bn': 'bn_IN', 'zh_HK': 'zh_CN'}.get(locale, locale) iso639 = self.j(self.TRANSLATIONS, 'iso_639', '%s.po' % iscpo) if os.path.exists(iso639) and self.check_iso639(iso639): dest = self.j(self.d(dest), 'iso639.mo') if self.newer(dest, iso639): jobs.append(['msgfmt', '-o', dest, iso639]) elif locale not in { 'en_GB', 'en_CA', 'en_AU', 'si', 'ur', 'sc', 'ltg', 'nds', 'te', 'yi', 'fo', 'sq', 'ast', 'ml', 'ku', 'fr_CA', 'him', 'jv', 'ka', 'fur', 'ber', 'my', 'fil', 'hy', 'ug' }: self.warn('No ISO 639 translations for locale:', locale) ln = normalize_locale(locale).partition('.')[0] if ln in lcdata: ld = lcdata[ln] lcdest = self.j(self.d(dest), 'lcdata.pickle') with open(lcdest, 'wb') as lcf: lcf.write(cPickle.dumps(ld, -1)) self.info('\nCompiling %d translation files...' % len(jobs)) tuple(parallel_check_output(jobs, self.info)) if self.iso639_errors: for err in self.iso639_errors: print(err) raise SystemExit(1) self.write_stats() self.freeze_locales() self.compile_user_manual_translations()
def write_stats(self): files = self.po_files() dest = self.stats if not self.newer(dest, files): return self.info('Calculating translation statistics...') stats = {} jobs = (['msgfmt', '--statistics', '-o', os.devnull, x] for x in files) for f, line in zip(files, parallel_check_output(jobs, self.info)): nums = tuple(map(int, re.findall(r'\d+', line))) trans = nums[0] total = trans if len(nums) == 1 else (trans + nums[1]) locale = self.mo_file(f)[0] stats[locale] = min(1.0, float(trans) / total) cPickle.dump(stats, open(dest, 'wb'), -1)
def compile_group(self, files, handle_stats=None, file_ok=None, action_per_file=None): from calibre.constants import islinux jobs, ok_files = [], [] hashmap = {} def stats_cache(src, data=None): cname = self.cache_name(src) + '.stats.json' with open(self.j(self.cache_dir, cname), ('rb' if data is None else 'wb')) as f: if data is None: return json.loads(f.read()) data = json.dumps(data) if not isinstance(data, bytes): data = data.encode('utf-8') f.write(data) for src, dest in files: base = os.path.dirname(dest) if not os.path.exists(base): os.makedirs(base) data, h = self.hash_and_data(src) current_hash = h.digest() saved_hash, saved_data = self.read_cache(src) if current_hash == saved_hash: with open(dest, 'wb') as d: d.write(saved_data) if handle_stats is not None: handle_stats(src, stats_cache(src)) else: if file_ok is None or file_ok(data, src): self.info('\t' + os.path.relpath(src, self.j(self.d(self.SRC), 'translations'))) if islinux: msgfmt = ['msgfmt'] else: msgfmt = [sys.executable, self.j(self.SRC, 'calibre', 'translations', 'msgfmt.py')] jobs.append(msgfmt + ['--statistics', '-o', dest, src]) ok_files.append((src, dest)) hashmap[src] = current_hash if action_per_file is not None: action_per_file(src) for (src, dest), line in zip(ok_files, parallel_check_output(jobs, self.info)): self.write_cache(open(dest, 'rb').read(), hashmap[src], src) nums = tuple(map(int, re.findall(r'\d+', line))) stats_cache(src, nums) if handle_stats is not None: handle_stats(src, nums)
def write_stats(self): files = self.po_files() dest = self.stats if not self.newer(dest, files): return self.info('Calculating translation statistics...') stats = {} jobs = ( ['msgfmt', '--statistics', '-o', os.devnull, x] for x in files ) for f, line in zip(files, parallel_check_output(jobs, self.info)): nums = tuple(map(int, re.findall(r'\d+', line))) trans = nums[0] total = trans if len(nums) == 1 else (trans + nums[1]) locale = self.mo_file(f)[0] stats[locale] = min(1.0, float(trans)/total) cPickle.dump(stats, open(dest, 'wb'), -1)
def run(self, opts): self.compile_content_server_translations() l = {} exec(compile(open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'lc_data.py')) .read(), os.path.join(os.path.dirname(os.path.abspath(__file__)), 'lc_data.py'), 'exec'), l, l) lcdata = {k:{k1:v1 for k1, v1 in v} for k, v in l['data']} self.iso639_errors = [] jobs = [] for f in self.po_files(): locale, dest = self.mo_file(f) base = os.path.dirname(dest) if not os.path.exists(base): os.makedirs(base) jobs.append(['msgfmt', '-o', dest, f]) iscpo = {'bn':'bn_IN', 'zh_HK':'zh_CN'}.get(locale, locale) iso639 = self.j(self.TRANSLATIONS, 'iso_639', '%s.po'%iscpo) if os.path.exists(iso639) and self.check_iso639(iso639): dest = self.j(self.d(dest), 'iso639.mo') if self.newer(dest, iso639): jobs.append(['msgfmt', '-o', dest, iso639]) elif locale not in { 'en_GB', 'en_CA', 'en_AU', 'si', 'ur', 'sc', 'ltg', 'nds', 'te', 'yi', 'fo', 'sq', 'ast', 'ml', 'ku', 'fr_CA', 'him', 'jv', 'ka', 'fur', 'ber', 'my', 'fil', 'hy', 'ug'}: self.warn('No ISO 639 translations for locale:', locale) ln = normalize_locale(locale).partition('.')[0] if ln in lcdata: ld = lcdata[ln] lcdest = self.j(self.d(dest), 'lcdata.pickle') with open(lcdest, 'wb') as lcf: lcf.write(cPickle.dumps(ld, -1)) self.info('\nCompiling %d translation files...' % len(jobs)) tuple(parallel_check_output(jobs, self.info)) if self.iso639_errors: for err in self.iso639_errors: print (err) raise SystemExit(1) self.write_stats() self.freeze_locales() self.compile_user_manual_translations()
def compile_user_manual_translations(self): self.info('Compiling user manual translations...') srcbase = self.j(self.d(self.SRC), 'translations', 'manual') destbase = self.j(self.d(self.SRC), 'manual', 'locale') complete = {} for x in os.listdir(srcbase): q = self.j(srcbase, x) if not os.path.isdir(q): continue dest = self.j(destbase, x, 'LC_MESSAGES') if os.path.exists(dest): shutil.rmtree(dest) os.makedirs(dest) jobs = [] for po in os.listdir(q): if not po.endswith('.po'): continue jobs.append([ 'msgfmt', '--statistics', '-o', self.j(dest, po.rpartition('.')[0] + '.mo'), self.j(q, po) ]) stats = tuple(parallel_check_output(jobs, self.info)) translated = untranslated = 0 for line in stats: m = re.search('(\d+).+(\d+)', line) if m is None: translated += int(re.search('\d+', line).group()) else: translated += int(m.group(1)) untranslated += int(m.group(2)) stats = {'translated': translated, 'untranslated': untranslated} with open(self.j(self.d(dest), 'stats.json'), 'wb') as f: json.dump(stats, f) total = translated + untranslated if total and (translated / float(total)) > 0.75: complete[x] = stats with open(self.j(destbase, 'completed.json'), 'wb') as f: json.dump(complete, f)
def compile_user_manual_translations(self): self.info('Compiling user manual translations...') srcbase = self.j(self.d(self.SRC), 'translations', 'manual') destbase = self.j(self.d(self.SRC), 'manual', 'locale') complete = {} for x in os.listdir(srcbase): q = self.j(srcbase, x) if not os.path.isdir(q): continue dest = self.j(destbase, x, 'LC_MESSAGES') if os.path.exists(dest): shutil.rmtree(dest) os.makedirs(dest) jobs = [] for po in os.listdir(q): if not po.endswith('.po'): continue jobs.append([ 'msgfmt', '--statistics', '-o', self.j(dest, po.rpartition('.')[0] + '.mo'), self.j(q, po) ]) stats = tuple(parallel_check_output(jobs, self.info)) translated = untranslated = 0 for line in stats: nums = tuple(map(int, re.findall(r'\d+', line))) translated += nums[0] if len(nums) > 1: untranslated += nums[1] stats = {'translated': translated, 'untranslated': untranslated} with open(self.j(self.d(dest), 'stats.json'), 'wb') as f: json.dump(stats, f) total = translated + untranslated # Raise the 30% threshold in the future if total and (translated / float(total)) > 0.3: complete[x] = stats with open(self.j(destbase, 'completed.json'), 'wb') as f: json.dump(complete, f, indent=True, sort_keys=True)
def compile_user_manual_translations(self): self.info('Compiling user manual translations...') srcbase = self.j(self.d(self.SRC), 'translations', 'manual') destbase = self.j(self.d(self.SRC), 'manual', 'locale') complete = {} for x in os.listdir(srcbase): q = self.j(srcbase, x) if not os.path.isdir(q): continue dest = self.j(destbase, x, 'LC_MESSAGES') if os.path.exists(dest): shutil.rmtree(dest) os.makedirs(dest) jobs = [] for po in os.listdir(q): if not po.endswith('.po'): continue jobs.append([ 'msgfmt', '--statistics', '-o', self.j( dest, po.rpartition('.')[0] + '.mo'), self.j(q, po)]) stats = tuple(parallel_check_output(jobs, self.info)) translated = untranslated = 0 for line in stats: m = re.search('(\d+).+(\d+)', line) if m is None: translated += int(re.search('\d+', line).group()) else: translated += int(m.group(1)) untranslated += int(m.group(2)) stats = {'translated':translated, 'untranslated':untranslated} with open(self.j(self.d(dest), 'stats.json'), 'wb') as f: json.dump(stats, f) total = translated + untranslated if total and (translated / float(total)) > 0.75: complete[x] = stats with open(self.j(destbase, 'completed.json'), 'wb') as f: json.dump(complete, f)
def compile_user_manual_translations(self): self.info('Compiling user manual translations...') srcbase = self.j(self.d(self.SRC), 'translations', 'manual') destbase = self.j(self.d(self.SRC), 'manual', 'locale') complete = {} for x in os.listdir(srcbase): q = self.j(srcbase, x) if not os.path.isdir(q): continue dest = self.j(destbase, x, 'LC_MESSAGES') if os.path.exists(dest): shutil.rmtree(dest) os.makedirs(dest) jobs = [] for po in os.listdir(q): if not po.endswith('.po'): continue jobs.append([ 'msgfmt', '--statistics', '-o', self.j( dest, po.rpartition('.')[0] + '.mo'), self.j(q, po)]) stats = tuple(parallel_check_output(jobs, self.info)) translated = untranslated = 0 for line in stats: nums = tuple(map(int, re.findall(r'\d+', line))) translated += nums[0] if len(nums) > 1: untranslated += nums[1] stats = {'translated':translated, 'untranslated':untranslated} with open(self.j(self.d(dest), 'stats.json'), 'wb') as f: json.dump(stats, f) total = translated + untranslated # Raise the 30% threshold in the future if total and (translated / float(total)) > 0.3: complete[x] = stats with open(self.j(destbase, 'completed.json'), 'wb') as f: json.dump(complete, f, indent=True, sort_keys=True)