def import_new_entries(self):
for root, dirs, files in os.walk(settings.EXTRACT_DIRS):
for some_file in files:
if some_file.endswith(".po"):
if os.path.normpath(os.path.join(os.path.join(root, some_file), "../..")).endswith("fr"):
polib_po = polib.pofile(os.path.join(root, some_file))
for e in polib_po:
occurrences = json.dumps(e.occurrences)
new_entry = MixedEntry.get_or_create(some_file,
e.msgid,
occurrences,
e.msgctxt,
e.msgid_plural)
msgstr_plural = json.dumps(e.msgstr_plural)
new_entry.add_french_terms(e.msgstr, msgstr_plural)
if os.path.normpath(os.path.join(os.path.join(root, some_file), "../..")).endswith("rw"):
polib_po = polib.pofile(os.path.join(root, some_file))
for e in polib_po:
occurrences = json.dumps(e.occurrences)
new_entry = MixedEntry.get_or_create(some_file,
e.msgid,
occurrences,
e.msgctxt,
e.msgid_plural)
msgstr_plural = json.dumps(e.msgstr_plural)
new_entry.add_rwanda_terms(e.msgstr, msgstr_plural)
def test_export_po_file_with_empty_msgstr_in_target_lang(self):
client = Client()
user = User.objects.create(username='******')
user.set_password('Admin')
user.save()
client.login(username='******', password='******')
new_lang = Language.objects.create(name='Japan', code='jp')
ProjectLanguage.objects.create(lang=new_lang, project=self.project)
path = os.path.join(PATH, 'data/django1.po')
import_po_file(path, self.project.id, self.project_language.id)
SetMessage.objects.filter(msgid='test.empty', lang=self.project.lang.id).update(msgstr='NonEmpty')
response = client.get('/project/%s/export/%s/' % (self.project.id, new_lang.id))
po = polib.pofile(response.content)
all_messages = SetMessage.objects.filter(
message_set__project_id=self.project.id,
lang=self.project.lang.id)
self.assertFalse(SetMessage.objects.get(msgid='test.empty', lang=new_lang.id).msgstr)
self.assertEqual(dict((i.msgid, unicode(i.msgstr, 'utf8')) for i in po)['test.empty'], 'NonEmpty')
SetMessage.objects.filter(msgid='test.empty').update(is_translated=True)
response = client.get('/project/%s/export/%s/' % (self.project.id, new_lang.id))
po = polib.pofile(response.content)
all_messages = SetMessage.objects.filter(message_set__project_id=self.project.id, lang=self.project.lang.id)
self.assertFalse(SetMessage.objects.get(msgid='test.empty', lang=new_lang.id).msgstr)
self.assertFalse(dict((i.msgid, unicode(i.msgstr, 'utf8')) for i in po)['test.empty'])
def main(templ, trans, outf):
print('Converting %s, %s' % (templ, trans), file=sys.stderr)
templ_po = polib.pofile(templ)
trans_po = polib.pofile(trans)
for entry in trans_po:
msgid = entry.msgid
msgid_plural = entry.msgid_plural
msgstr = entry.msgstr
orig = templ_po.find(msgid).msgstr
orig_plural = templ_po.find(msgid).msgstr_plural
if msgid_plural:
newid = msgid + '|' + msgid_plural
entry.msgctxt = newid
entry.msgid = orig_plural['0']
entry.msgid_plural = orig_plural['1']
if entry.msgstr_plural == orig_plural:
entry.msgstr_plural = dict()
entry.msgstr_plural[0] = ''
entry.msgstr_plural[1] = ''
else:
entry.msgctxt = msgid
entry.msgid = orig
if msgstr == orig:
entry.msgstr = ''
trans_po.save(outf)
def test_export_po_file_with_two_lang_check_record_number(self):
client = Client()
user = User.objects.create(username='******')
user.set_password('Admin')
user.save()
client.login(username='******', password='******')
response = client.get('/project/%s/export/%s/' % (self.project.id, self.project_language.id))
self.assertEqual(response.status_code, 200)
po = polib.pofile(response.content)
self.assertEqual(
len(SetMessage.objects.filter(
message_set__project_id=self.project.id,
lang=self.project_language.id)),
len(po.translated_entries()))
new_lang = Language.objects.create(name='Russian', code='ru')
ProjectLanguage.objects.create(lang=new_lang, project=self.project)
path = os.path.join(PATH, 'data/django.po')
import_po_file(path, self.project.id, self.project_language.id)
response = client.get('/project/%s/export/%s/' % (self.project.id, new_lang.id))
po = polib.pofile(response.content)
all_messages = SetMessage.objects.filter(
message_set__project_id=self.project.id,
lang=new_lang.id)
self.assertEqual(len(all_messages), len(po.translated_entries()))
self.assertTrue(
dict((i.msgid, unicode(i.msgstr, 'utf8')) for i in po) ==
dict((i.msgid, i.msgstr) for i in all_messages))
self.assertTrue(
set(i.msgid for i in all_messages) == set(i.msgid for i in po))
def process_po(lang_id):
new_fn = lang_id + '_new.po'
old_fn = lang_id + '.po'
po_new = pofile(new_fn)
po_old = pofile(old_fn)
msgids = []
for new_entry in po_new.translated_entries():
msgids.append([new_entry.msgctxt, new_entry.msgid])
for old_entry in po_old.translated_entries():
if 'translator-credits' == new_entry.msgid:
msgids.pop()
break
if new_entry.msgctxt == old_entry.msgctxt and \
new_entry.msgid == old_entry.msgid and \
new_entry.msgstr == old_entry.msgstr:
msgids.pop()
break
names = []
for (msgctxt, msgid) in msgids:
print "looking for msgctxt '%s' msgid '%s' for lang '%s'" % \
(msgctxt, msgid, lang_id)
names = names + who_translated(lang_id, msgctxt, msgid)
lang_name = get_lang_name(po_new, lang_id)
if len(set(names)) == 1:
author = po_new.metadata['Last-Translator']
else:
author = ', '.join(set(names)) + ' <multiple-authors>'
cmd = 'git commit %s.po -m "Update %s translation thanks to %s" --author "%s"' % \
(lang_id, lang_name, ', '.join(set(names)), author)
return cmd
def make_messages(src_path, path, name, outpath=None):
sys.argv = [None, '-a', '-d', name, '-p', path]
for root, dirs, files in os.walk(src_path):
for f in files:
if f.endswith('.py'):
p = os.path.join(root, f)
sys.argv.append(p)
gtext()
wzr_filename = os.path.join(path, name+'.pot')
for pos in os.scandir(path):
if pos.is_dir():
lang = pos.name
ftmp = os.path.join(path, lang)
if outpath:
ftmp = os.path.join(ftmp, outpath)
filename = os.path.join(ftmp, name + '.po')
old_filename = filename.replace('.po', '.bak')
mo_filename = filename.replace('.po', '.mo')
try:
os.remove(old_filename)
except:
pass
try:
os.rename(filename, old_filename)
except:
pass
wzr = polib.pofile(wzr_filename)
po = polib.pofile(old_filename)
po.merge(wzr)
po.save(filename)
po.save_as_mofile(mo_filename)
def main():
parser = argparse.ArgumentParser(description='Check for mismatches in msgid and msgstr with respect to numerous formats: including file names, HTML tags, python gettext variables, and My Opera Mail variables. When no optional arguments are given, the default behavior is to check for HTML and python gettext errors.')
parser.add_argument('po_file', nargs='+', help='PO file whose msgstr should be checked, or a newline-separated list of PO files')
parser.add_argument("--all", help="severe checking, with all arguments", action="store_true")
parser.add_argument("--extensions", help="check file name extensions in strings", action="store_true")
parser.add_argument("--filenames", help="check file names in strings", action="store_true")
parser.add_argument("--html", help="check HTML tags in strings", action="store_true")
parser.add_argument("--urls", help="check URLs in strings", action="store_true")
parser.add_argument("--momail", help="check My Opera Mail variables in strings", action="store_true")
parser.add_argument("--products", help="check product names in strings", action="store_true")
parser.add_argument("--python", help="check python gettext variables in strings", action="store_true")
parser.add_argument("--msgctxt", help="check a single string ID, not the whole PO file", action="store")
parser.add_argument("--strings", help="check a list of string IDs, not the whole PO file", action="store")
args = parser.parse_args()
for name in args.po_file:
try:
find_mismatched(polib.pofile(name), args)
except:
for line in open(name).readlines():
try:
path = line.strip(' \n')
find_mismatched(polib.pofile(path), args)
except IOError:
sys.exit('File %s is not a PO file or a list of PO files' % name)
return None
def insert_translations(args):
"""
Insert strings from a source PO into the target PO file
"""
source = polib.pofile(args.source_po)
target = polib.pofile(args.target_po, wrapwidth=0)
for line in open(args.strings).readlines():
string = line.strip(' \n')
source_entry = source.find(string, by='msgctxt')
target_entry = target.find(string, by='msgctxt')
if source_entry and target_entry:
if not target_entry.msgstr:
print string + color('31', ' untranslated')
target_entry.msgstr = source_entry.msgstr
target_entry = remove_fuzzy(target_entry)
continue
source_entry = source.find(string, by='msgid')
target_entry = target.find(string, by='msgid')
if source_entry and target_entry:
if not target_entry.msgstr:
print string + color('31', ' untranslated')
target_entry.msgstr = source_entry.msgstr
target_entry = remove_fuzzy(target_entry)
continue
target.save()
return None
def create_po_template(self):
django = polib.pofile(str(Path(self.locale_paths[0]).joinpath('django.pot')), check_for_duplicates = True)
djangojs = polib.pofile(str(Path(self.locale_paths[0]).joinpath('djangojs.pot')), check_for_duplicates = True)
self.add_po_domain(django, 'django')
self.add_po_domain(djangojs, 'djangojs')
for entry in djangojs:
django.append(entry)
django.header = "\nGNOME Shell extensions repository\n"
django.header += "\n"
django.header += "DO NOT EDIT!\n"
django.header += "This file is auto generated with manage.py makemessages."
django.header += "\n"
django.metadata = {
'Project-Id-Version': '1.0',
'Report-Msgid-Bugs-To': '*****@*****.**',
'POT-Creation-Date': datetime.now(pytz.utc).strftime('%Y-%m-%d %H:%M%z'),
'MIME-Version': '1.0',
'Content-Type': 'text/plain; charset=utf-8',
'Content-Transfer-Encoding': '8bit',
}
django.sort()
django.save(Path(self.po_path).joinpath("extensions-web.pot"))
def test_sample_data(self):
work_file = WORK / "django.po"
shutil.copyfile(TEST_DATA / "django_before.po", work_file)
original_pofile = polib.pofile(work_file)
written = segment_pofile(
work_file,
{
'studio.po': [
'cms/*',
'other_cms/*',
],
}
)
self.assertEqual(written, set([WORK / "django.po", WORK / "studio.po"]))
pofiles = [polib.pofile(f) for f in written]
after_entries = sum(len(pofile) for pofile in pofiles)
self.assertEqual(len(original_pofile), after_entries)
original_ids = set(m.msgid for m in original_pofile)
after_ids = set(m.msgid for pofile in pofiles for m in pofile)
self.assertEqual(original_ids, after_ids)
self.assert_pofile_same(WORK / "django.po", TEST_DATA / "django_after.po")
self.assert_pofile_same(WORK / "studio.po", TEST_DATA / "studio.po")
def diff_one_file(fname, canon_name):
po = polib.pofile(fname)
canonical_po = polib.pofile(canon_name)
diff_po = polib.POFile()
po.merge(canonical_po)
# msgids in the def. po file with no cananonical definition are
# marked as obsolute after a merge
for entry in po.obsolete_entries():
entry.msgstr = ''
entry.obsolete = False
diff_po.append(entry)
# Add any msgids that are untranslated in the
# canonical catalog as well
for entry in canonical_po.untranslated_entries():
entry.msgstr = ''
diff_po.append(entry)
full_fname_path = os.path.abspath(fname)
fname_dir = os.path.dirname(full_fname_path)
fname_name = os.path.basename(full_fname_path).replace('.po', '') # no extension
diff_po_path = os.path.join(fname_dir, '%s_diff.po' % fname_name)
diff_po.save(diff_po_path)
return "Created %s with %s translations" % (diff_po_path, len(diff_po))
def _get_new(self, filepath1, filepath2):
""" Compare two .po or .pot files and find all the entries in the
second file that are not in the first one.
Optionally:
- include fuzzy and dirty non-new entries.
- ignore new but translated entries.
- prefill msgstr of all entries with their default values
"""
firstpo = polib.pofile(filepath1)
secondpo = polib.pofile(filepath2)
entries = []
for new_entry in secondpo:
if new_entry.obsolete:
# Ignore outcommented entries
continue
old_entry = firstpo.find(new_entry.msgid)
if not old_entry:
# We have a new entry
if self.options.ignore_translated and new_entry.translated():
continue
entries.append(new_entry)
elif (
self.options.untranslated
and not new_entry.translated()
or self.options.fuzzy
and "fuzzy" in new_entry.flags
or self.options.dirty
and self.is_dirty(old_entry, new_entry)
):
entries.append(new_entry)
return entries
def i18n_make_updates(is_js, suffix):
with working_directory(ARABIC_LOCALE_DIR):
django_edx = polib.pofile('django{}.po'.format(suffix))
django_latest = polib.pofile('latest-django{}.po'.format(suffix))
edx_entries = {}
for entry in django_edx.translated_entries():
edx_entries[entry.msgid] = entry
for entry in reversed(django_latest):
if not entry.translated():
django_latest.remove(entry)
else:
edx_entry = edx_entries.get(entry.msgid)
if edx_entry and edx_entry.msgstr == entry.msgstr:
# Remove the ones that didn't change
django_latest.remove(entry)
print 'Added {} updated translations for django{}.po:'.format(
len(django_latest),
suffix,
)
django_latest.save('django{}-updates.po'.format(suffix))
path('latest-django{}.po'.format(suffix)).remove()
def mergepo_polib(target, source, options):
changed = 0
po1 = polib.pofile(target)
po2 = polib.pofile(source)
if options.overwrite:
for entry in po1.entries():
other = po2.find(entry.msgid, include_obsolete_entries=True)
if not other:
continue
if options.nonnull and other.msgstr == other.msgid:
continue
if other.translated() and other.msgstr != entry.msgstr:
entry.msgstr = other.msgstr
if 'fuzzy' in other.flags:
if not 'fuzzy' in entry.flags:
entry.flags.append('fuzzy')
else:
if 'fuzzy' in entry.flags:
entry.flags.remove('fuzzy')
changed = changed + 1
else:
for entry in po1.untranslated_entries():
other = po2.find(entry.msgid, include_obsolete_entries=True)
if not other:
continue
if options.nonnull and other.msgstr == other.msgid:
continue
if other.translated():
entry.msgstr = other.msgstr
changed = changed + 1
if changed > 0:
po1.save(target)
return changed
def merge_po_files(po_files, fuzzy=False):
"""Find each po file from the current directory, group them by name,
and replicate known translations from each one to the others.
"""
"""Given a list of po files, replicate known translation from each
file to the others.
"""
known_translations = {}
# Aggregate all known translations
for po_file in tqdm(po_files, desc="Searching known translations"):
po_file = polib.pofile(po_file)
for entry in po_file:
if 'fuzzy' not in entry.flags and entry.msgstr != '':
known_translations[entry.msgid] = entry.msgstr
# Propagate them
done = 0
for po_file in tqdm(po_files, desc="Replicating them"):
po_file = polib.pofile(po_file)
for entry in po_file:
if entry.msgid in known_translations:
entry.msgstr = known_translations[entry.msgid]
elif fuzzy:
best_match = find_best_match(list(known_translations.keys()),
entry.msgid)
if best_match is not None:
print("I think\n {}\n =\n {}".format(entry.msgid,
best_match))
entry.msgstr = known_translations[best_match]
entry.flags.append('fuzzy')
po_file.save()
def __call__(self, value):
try:
if not os.path.isfile(value):
raise IOError()
polib.pofile(value)
except IOError:
raise ValidationError(self.message)
return value
def main():
options = parse_args()
language_files = get_language_files(options.locale_dir, options.force_create)
pot_file = polib.pofile(options.pot_file)
warning_file = open('warning.log', 'w')
for lfile in language_files:
to_translate = polib.pofile(lfile)
language = guess_language(lfile)
po_list = get_files_to_translate(lfile)
if language == 'en':
continue
to_translate.merge(pot_file)
to_translate.save()
#~ Now we merge in any new files
for new_trans in po_list:
os.system('/usr/bin/msgmerge %s %s -o %s' % (new_trans, lfile, lfile))
to_translate = polib.pofile(lfile)
total = len(to_translate.untranslated_entries())
count = 1
#~ Translate with Google Translate
print '--- Working on %s : %d entries to translate.' % (language.upper(), total)
for entry in to_translate.untranslated_entries():
entry.msgstr = translate(entry.msgid.lower(), language, key=options.google_key)
entry.flags.append(u'fuzzy')
print '%s [%d/%d] ...%s... --> ...%s...' % (language, count, total, entry.msgid[:10], entry.msgstr[:10])
count += 1
#~ Clean up msgstr entries
total = len(to_translate)
count = 1
for entry in to_translate:
tmp_msgstr = entry.msgstr
entry.msgstr = clean_msgstr(entry.msgstr)
if tmp_msgstr != entry.msgstr:
print 'Cleaned: %s --> %s' % (tmp_msgstr, entry.msgstr)
if nefarious_regex.search(entry.msgstr):
match = nefarious_regex.match(entry.msgstr).group(1)
print 'Warning: Found improper match %s.' % match
warning_log.write('Warning: Found improper match %s in %s' % (match, entry.msgstr))
to_translate.save()
if options.save_as_mo:
filename, suffix = lfile.rsplit('.', 1)
mo_file = filename + '.mo'
os.system('/usr/bin/rm %s -f' % mo_file)
print 'Saving mo as %s' % mo_file
os.system('msgfmt -o %s --use-fuzzy %s' % (mo_file, filename))
def _update_po(self, target_path, source_path):
log.info("Updating %s from %s" % (target_path, source_path))
target = polib.pofile(target_path)
source = polib.pofile(source_path)
target_dict = {}
for entry in target:
target_dict[entry.msgid] = entry
count = 0
for entry in source:
if not entry.msgid in target_dict:
if self.options.insert_new:
# If it's not in the target and -n is true, add it:
target.append(entry)
continue
target_entry = target_dict[entry.msgid]
if target_entry.msgstr != entry.msgstr:
# The messages are different. Check if we should update.
if not self.options.force:
# if force is set, skip the checks and always update.
if not entry.msgstr:
# The new message is empty, we should not update
continue
if not utils.msg_is_updated(entry):
# The new message is fuzzy
if utils.msg_is_updated(target_entry):
# the old message is not fuzzy and not empty
continue
elif not self.options.use_fuzzy:
# Never use fuzzy unless use_fuzzy is set:
continue
# Update the target
target_dict[entry.msgid].msgstr = entry.msgstr
# Since the target has been updated with a new translation,
# it is no longer fuzzy
if self.options.reset_fuzzy and \
'fuzzy' in target_dict[entry.msgid].flags:
target_dict[entry.msgid].flags.remove('fuzzy')
count += 1
if not count:
log.info("No entries updated")
return
# Update the revision date.
source_date = source.metadata.get('PO-Revision-Date', '')
target_date = target.metadata.get('PO-Revision-Date', '')
if source_date > target_date:
# In this case we use the source date as the last update
target.metadata['PO-Revision-Date'] = source_date
else:
# The target has been modified after the source.
# We set the revision date to now, to mark it as changed.
target.metadata['PO-Revision-Date'] = po_timestamp()
target.save(target_path)
log.info("%s entries updated" % count)
def merge_po_and_preserve(source, dest):
# Merges source entries into dest, but keep old entries intact
sourcepo = polib.pofile(source)
destpo = polib.pofile(dest)
for entry in sourcepo:
if destpo.find(entry.msgid) is not None:
# The entry is already there
continue
destpo.append(entry)
destpo.save()
def merge_pots_into_pos(folder):
# We're going to take all pot files in `folder` and for each lang, merge it with the po file
# with the same name.
potfiles = files_with_ext(folder, '.pot')
for potfile in potfiles:
refpot = polib.pofile(potfile)
refname = op.splitext(op.basename(potfile))[0]
for lang in get_langs(folder):
po = polib.pofile(op.join(folder, lang, LC_MESSAGES, refname + '.po'))
po.merge(refpot)
po.save()
def main():
parser = argparse.ArgumentParser()
#parser.add_argument('--no-ignore-comments')
parser.add_argument('old_file')
parser.add_argument('new_file')
args = parser.parse_args()
diffs = podiff(polib.pofile(args.old_file), polib.pofile(args.new_file))
pprint_diff(diffs)
return exit_code(diffs)
def update(self):
if not getattr(self, "pot", None):
self.pot = polib.pofile(self.potname)
for file in os.listdir("po"):
po = polib.pofile(os.path.join("po", file))
po.merge(self.pot)
po.metadata["Project-Id-Version"] = self.pot.metadata["Project-Id-Version"]
po.metadata["POT-Creation-Date"] = self.pot.metadata["POT-Creation-Date"]
po.save()
print "PO files updated"
def test_files(self):
po_file_list = get_file_list(os.path.join(TEST_PATH, "temp_files", "docs"),["po","pot"])
po_file_list.sort()
write_po_files(po_file_list,os.path.join(TEST_PATH, "test_files", "doc_filler.txt"))
f1 = polib.pofile(os.path.join(TEST_PATH, "temp_files", "docs", "aggregation.po"))
f2 = polib.pofile(os.path.join(TEST_PATH, "test_files", "filled_docs", "aggregation.po"))
for l1, l2 in zip(f1, f2):
self.assertEqual(l1, l2)
f1 = polib.pofile(os.path.join(TEST_PATH, "temp_files", "docs", "administration.po"))
f2 = polib.pofile(os.path.join(TEST_PATH, "test_files", "filled_docs", "administration.po"))
for l1, l2 in zip(f1, f2):
self.assertEqual(l1, l2)
def GenerateStringTables(out_dir, pot_file, po_files):
"""Generates string tables from .po files.
Args:
out_dir: The directory to put the string table files in.
pot_file: The .pot file containing the master list of strings.
po_files: A list of .po files to generate string tables from.
"""
# first build map of string -> id for the master table
master_table = {}
master_strings = []
pot_file = polib.pofile(pot_file)
for entry in pot_file:
master_strings.append(entry.msgid)
master_table[entry.msgid] = len(master_strings)-1
# if no .po files were given (just the .pot file), generate the master table
if not po_files:
# write out master table to source file
WriteStringTable(master_strings, 'master_string_table',
None, out_dir, 'master.po.cc')
# now write out a table for each locale
locales = []
for po_file in po_files:
# extract the name of the locale from the filename ("fr.po" -> "fr")
locale_name = os.path.splitext(os.path.basename(po_file))[0]
locales.append(locale_name)
if not os.path.exists(po_file):
print >> sys.stderr, 'error: %s does not exist!' % po_file
sys.exit(1)
po = polib.pofile(po_file)
translation_table = {}
for entry in po:
translation_table[entry.msgid] = entry.msgstr
locale_strings = []
for ms in master_strings:
if ms in translation_table:
locale_strings.append(translation_table[ms])
else:
locale_strings.append('')
if len(locale_strings) != len(master_strings):
print >> sys.stderr, ('warning: %s doesn\'t match master .pot file' %
po_file)
WriteStringTable(locale_strings, '%s_string_table' % locale_name,
locale_name, out_dir, '%s.po.cc' % locale_name,
comments=master_strings)
def make_messages(src_path, path, name, outpath=None, ext_locales=[]):
backup_argv = sys.argv
sys.argv = [None, '-a', '-d', name, '-p', path]
for root, dirs, files in os.walk(src_path):
for f in files:
if f.endswith('.py'):
p = os.path.join(root, f)
sys.argv.append(p)
gtext()
wzr_filename = os.path.join(path, name+'.pot')
for pos in os.scandir(path):
if pos.is_dir():
lang = pos.name
ftmp = os.path.join(path, lang)
if outpath:
ftmp = os.path.join(ftmp, outpath)
filename = os.path.join(ftmp, name + '.po')
old_filename = filename.replace('.po', '.bak')
mo_filename = filename.replace('.po', '.mo')
try:
os.remove(old_filename)
except:
pass
try:
os.rename(filename, old_filename)
except:
pass
wzr = polib.pofile(wzr_filename)
if os.path.exists(old_filename):
po = polib.pofile(old_filename)
else:
po = polib.POFile()
po.merge(wzr)
#for pos2 in ext_locales:
# ftmp = os.path.join(pos2[1],lang)
# if outpath:
# ftmp = os.path.join(ftmp, outpath)
# ext_filename = os.path.join(ftmp, name+'.po')
# if os.path.exists(ext_filename):
# ext_po = polib.pofile(ext_filename)
# po.merge(ext_po)
po.save(filename)
po.save_as_mofile(mo_filename)
sys.argv = backup_argv
def i18n_edraak_push(is_js, suffix):
"""
Extracts strings and appends it to the provided .PO file.
It searches for translation strings that are marked
with "# Translators: " comment.
"""
edx_pofile = polib.pofile('conf/locale/ar/LC_MESSAGES/django{}.po'.format(suffix))
with open("conf/locale/config.yaml", 'r') as locale_config_file:
locale_config = yaml.load(locale_config_file)
partial_pofiles = locale_config['generate_merge']['django{}.po'.format(suffix)]
with working_directory('conf/locale/en/LC_MESSAGES'):
edraak_specific_path = path('edraak{}-platform.po'.format(suffix))
if edraak_specific_path.exists():
edraak_specific_path.unlink()
edraak_specific = polib.POFile()
edraak_specific.metadata = {
'Project-Id-Version': 'Edraak 1',
'Report-Msgid-Bugs-To': '*****@*****.**',
'POT-Creation-Date': '2014-12-15 11:17+0200',
'PO-Revision-Date': 'YEAR-MO-DA HO:MI+ZONE',
'Last-Translator': 'Edraak Dev <*****@*****.**>',
'Language-Team': 'Edraak Dev <*****@*****.**>',
'MIME-Version': '1.0',
'Content-Type': 'text/plain; charset=utf-8',
'Content-Transfer-Encoding': '8bit',
'Generated-By': 'Paver',
}
for po_path in partial_pofiles:
print 'processing', po_path
pofile = polib.pofile(po_path)
for entry in pofile:
new_entry = entry not in edx_pofile
marked_as_specific = 'edraak-specific' in entry.comment.lower()
if new_entry or marked_as_specific:
edraak_specific.append(entry)
edraak_specific.save(edraak_specific_path)
sh('tx push -l en -s -r edraak.edraak{}-platform'.format(suffix))
def convert(self, value, param, ctx):
if not os.path.exists(value) and ':' in value:
# The user passed a <locale>:<path> value
(locale, path) = value.split(':', 1)
path = os.path.expanduser(path)
real_path = super(CatalogFile, self).convert(path, param, ctx)
return (locale, polib.pofile(real_path))
else:
real_path = super(CatalogFile, self).convert(value, param, ctx)
catalog = polib.pofile(real_path)
locale = catalog.metadata.get('Language')
if not locale:
locale = os.path.splitext(os.path.basename(real_path))[0]
return (locale, catalog)
def translate_po(compendium,po):
"""
Simple translate tool
"""
compendium = polib.pofile(compendium)
traducir = polib.pofile(po)
temp_compendium=compendium
temp_compendium.merge(traducir)
potools.simple_po_creation("final.po")
potools.set_metadata_pilarized("final.po")
final_po=polib.pofile("final.po")
for entry in temp_compendium:
if not entry.obsolete:
final_po.append(entry)
final_po.save("final.po")
def translate_file(self, fname):
"""Translates the po file at fname
Note: The translation is done in-place and saved to the
given pofile.
"""
po = polib.pofile(fname)
# FIXME - This might be a bit goofy
po.metadata['Language'] = ",".join(self.pipeline_spec)
po.metadata['Plural-Forms'] = 'nplurals=2; plural= n != 1'
po.metadata['Content-Type'] = 'text/plain; charset=UTF-8'
count = 0
for entry in po:
if entry.msgid_plural:
entry.msgstr_plural['0'] = self.translate_string(
entry.msgid)
entry.msgstr_plural['1'] = self.translate_string(
entry.msgid_plural)
else:
entry.msgstr = self.translate_string(entry.msgid)
if 'fuzzy' in entry.flags:
entry.flags.remove('fuzzy') # clear the fuzzy flag
count += 1
print 'Munged %d messages in %s' % (count, fname)
po.save()
def check_pofile(po_file_path):
if not os.path.exists(po_file_path):
writeout("File not found %s\n" % po_file_path)
sys.exit(2)
po = polib.pofile(po_file_path)
UNTRANSLATED = 0
for entry in po.untranslated_entries():
UNTRANSLATED += 1
writeout("UNTRANSLATED\t", entry.msgid)
FUZZY = 0
for entry in po.fuzzy_entries():
FUZZY += 1
writeout("FUZZY\t", entry.msgid, entry.msgstr)
if FUZZY or UNTRANSLATED:
writeout("\n----------")
if FUZZY:
writeout("%d FUZZY string found." % FUZZY)
if UNTRANSLATED:
writeout("%d UNTRANSLATED string found." % UNTRANSLATED)
writeout("----------")
writeout("Edit: %s\n\n" % os.path.abspath(po_file_path))
sys.exit(100)
writeout("%s: PO File OK\n\n" % po_file_path)
sys.exit(0)
def test_strings_complete():
po = polib.pofile('locales/en/LC_MESSAGES/base.po')
assert po.percent_translated() == 100
import os
import pathlib
CURRENT_DIR = pathlib.Path(__file__).parent
trans_dir = CURRENT_DIR / "translated/pl/LC_MESSAGES"
mtg_exps = (f for f in os.listdir(trans_dir)
if f.endswith(".po") and len(f) == 6)
def percent_translated(obj):
total = len(
[e for e in obj if not e.obsolete and not e.msgid.startswith(":")])
if total == 0:
return 100
translated = len(obj.translated_entries())
return int(translated * 100 / float(total))
with open("percentages.inc", "w") as outfile:
for filename in mtg_exps:
expansion = filename[:3]
pofile = polib.pofile(trans_dir / filename)
percentage = (percent_translated(pofile) // 5 * 5) or 5
pie = f"pie{percentage:02d}" if percentage < 100 else "done"
outfile.write(f"""
.. |{expansion}_percent| image:: images/{pie}.png
""")
if percentage == 100:
outfile.write(" :alt: Tłumaczenie zakończone\n")
msgid=message_id.decode('utf-8'),
msgstr='',
occurrences=[(langfile, '')]
)
if message_id != '':
try:
pot.append(potentry)
except ValueError:
print 'The entry already exists'
pot.save('../../po/stellarium-desktop/stellarium-desktop.pot')
# Merge translations
for pofile in glob.glob(podir + '/*.po'):
lang = pofile[:-3].rsplit('/', 1)[1]
pofilename = pofile
po = polib.pofile(pofilename)
po.merge(pot)
po.save(pofilename)
for langfile in files:
# open desktop file
with open(langfile, "r") as deskfile:
text = deskfile.read()
with open(langfile, "w") as deskfile:
for transblock in tpattern.findall(text):
text = text.replace(transblock, '')
# Parse PO files
for pofile in sorted(glob.glob(podir + '/*.po'), reverse=True):
lang = pofile[:-3].rsplit('/', 1)[1]
pofilename = pofile
language_code = sys.argv[1]
else:
print("Usage seed_translation.py lang")
print(" where lang is your two letteer ISO-639-1 language code")
exit()
po_files_dir = './translated_po/%s' % language_code
files = os.listdir(po_files_dir)
translate_all = True
translator = Translator(to_lang=language_code)
for po_file_name in files:
print("TRANSLATE %s" % po_file_name)
po = polib.pofile(os.path.join(po_files_dir, po_file_name))
for entry in po:
if translate_all or entry.msgstr == '':
print(entry.msgid)
sentences = entry.msgid.split('.')
translated_sentences = []
for sentence in sentences:
try:
translated_sentences.append(translator.translate(sentence))
except:
print("ERROR TRANSLATING SENTENCE '%s'" % sentence)
translation = '. '.join(translated_sentences)
print(translation)
print("-" * 40)
entry.msgstr = translation.strip()
po.save(os.path.join(po_files_dir, po_file_name))
def get_context_data(self, **kwargs):
context = super(TranslationFormView, self).get_context_data(**kwargs)
entries = self.get_entries()
paginator = Paginator(entries, rosetta_settings.MESSAGES_PER_PAGE)
# Handle REF_LANG setting; mark up our entries with the reg lang's
# corresponding translations
LANGUAGES = list(rosetta_settings.ROSETTA_LANGUAGES)
if rosetta_settings.ENABLE_REFLANG:
if self.ref_lang_po_file:
for o in paginator.object_list:
ref_entry = self.ref_lang_po_file.find(o.msgid)
if ref_entry and ref_entry.msgstr:
o.ref_txt = ref_entry.msgstr
else:
o.ref_txt = o.msgid
else:
for o in paginator.object_list:
o.ref_txt = o.msgid
# XXX: having "MSGID" at the end of the dropdown is really odd, no?
# Why not instead do this?
# LANGUAGES = [('', '----')] + list(settings.LANGUAGES)
LANGUAGES.append(('msgid', 'MSGID'))
# Determine page number & how pagination links should be displayed
try:
page = int(self._request_request('page', 1))
except ValueError:
page = 1 # fall back to page 1
else:
if not (0 < page <= paginator.num_pages):
page = 1
needs_pagination = paginator.num_pages > 1
if needs_pagination:
if paginator.num_pages >= 10:
page_range = pagination_range(1, paginator.num_pages, page)
else:
page_range = range(1, 1 + paginator.num_pages)
rosetta_messages = paginator.page(page).object_list
# Handle MAIN_LANGUAGE setting, if applicable; mark up each entry
# in the pagination window with the "main language"'s string.
main_language_id = rosetta_settings.MAIN_LANGUAGE
main_language = None
if main_language_id and main_language_id != self.language_id:
# Translate from id to language name
for language in rosetta_settings.ROSETTA_LANGUAGES:
if language[0] == main_language_id:
main_language = _(language[1])
break
if main_language:
main_lang_po_path = self.po_file_path.replace(
'/%s/' % self.language_id, '/%s/' % main_language_id)
# XXX: brittle; what if this path doesn't exist? Isn't a .po file?
main_lang_po = pofile(main_lang_po_path)
for message in rosetta_messages:
message.main_lang = main_lang_po.find(message.msgid).msgstr
# Collect some constants for the template
rosetta_i18n_lang_name = six.text_type(
dict(rosetta_settings.ROSETTA_LANGUAGES).get(self.language_id))
# "bidi" as in "bi-directional"
rosetta_i18n_lang_bidi = self.language_id.split(
'-')[0] in settings.LANGUAGES_BIDI
query_string_args = {}
if self.msg_filter:
query_string_args['msg_filter'] = self.msg_filter
if self.query:
query_string_args['query'] = self.query
if self.ref_lang:
query_string_args['ref_lang'] = self.ref_lang
# Base for pagination links; the page num itself is added in template
pagination_query_string_base = urlencode_safe(query_string_args)
# Base for msg filter links; it doesn't make sense to persist page
# numbers in these links. We just pass in ref_lang, if it's set.
filter_query_string_base = urlencode_safe(
{k: v
for k, v in query_string_args.items() if k == 'ref_lang'})
context.update({
'version':
get_rosetta_version(),
'LANGUAGES':
LANGUAGES,
'rosetta_settings':
rosetta_settings,
'rosetta_i18n_lang_name':
rosetta_i18n_lang_name,
'rosetta_i18n_lang_code':
self.language_id,
'rosetta_i18n_lang_code_normalized':
self.language_id.replace('_', '-'),
'rosetta_i18n_lang_bidi':
rosetta_i18n_lang_bidi,
'rosetta_i18n_filter':
self.msg_filter,
'rosetta_i18n_write':
self.po_file_is_writable,
'rosetta_messages':
rosetta_messages,
'page_range':
needs_pagination and page_range,
'needs_pagination':
needs_pagination,
'main_language':
main_language,
'rosetta_i18n_app':
get_app_name(self.po_file_path),
'page':
page,
'query':
self.query,
'pagination_query_string_base':
pagination_query_string_base,
'filter_query_string_base':
filter_query_string_base,
'paginator':
paginator,
'rosetta_i18n_pofile':
self.po_file,
'ref_lang':
self.ref_lang,
})
return context
def get_pofile(self) -> polib.POFile:
with open(self.translation_filename(), "rb") as f:
content = f.read()
return polib.pofile(content.decode(), encoding="utf-8")
Newer browsers protect you better against viruses, scams and other threats. Outdated browsers have security holes which are fixed in updates.
If you can't change your browser because of compatibility issues, think about installing a second browser for browsing and keep the old one for the compatibility.
If you can't change your browser because of compatibility issues, think about installing a second browser for browsing and keep the old one for compatibility.
If you are on a computer that is maintained by an admin and you cannot install a new browser, ask your admin about it.
Ask your admin to update your browser if you cannot install updates yourself.
blaasdasdfsdaf
faselsdfsadf"""
pairs = pairs.replace("\r", "")[1:-1].split("\n\n")
mappings = {s.split("\n")[0]: s.split("\n")[1] for s in pairs}
#%%
po = polib.pofile('lang/de_DE/LC_MESSAGES/update.po')
valid_entries = [e for e in po if not e.obsolete]
for entry in valid_entries:
#print(entry.msgid)
if entry.msgid in mappings:
print("replacing", entry.msgid[:10], "with",
mappings[entry.msgid][:10])
entry.msgid = mappings[entry.msgid]
po.save()
po.save_as_mofile('lang/de_DE/LC_MESSAGES/update.mo')
#%%
pairs = """aaa
bbb
def prereleaser_before(data):
"""
1. Run the unit tests one last time before we make a release.
2. Update the CONTRIBUTORS.txt file.
Note: Install * polib (https://pypi.python.org/pypi/polib).
* pep8.
"""
print('Running unit tests.')
subprocess.check_output(["python", "example_project/manage.py", "test", "photologue"])
print('Running PEP8 check.')
# See setup.cfg for configuration options.
subprocess.check_output(["pep8"])
print('Checking that we have no outstanding DB migrations.')
output = subprocess.check_output(["python", "example_project/manage.py", "makemigrations", "--dry-run",
"photologue"])
if not output == b"No changes detected in app 'photologue'\n":
raise Exception('There are outstanding migrations for Photologue.')
print('Updating CONTRIBUTORS.txt')
# This command will get the author of every commit.
output = subprocess.check_output(["git", "log", "--format='%aN'"])
# Convert to a list.
contributors_list = [contributor.strip("'") for contributor in output.decode('utf-8').split('\n')]
# Now add info from the translator files. This is incomplete, we can only list
# the 'last contributor' to each translation.
for language in os.listdir('photologue/locale/'):
filename = 'photologue/locale/{0}/LC_MESSAGES/django.po'.format(language)
po = polib.pofile(filename)
last_translator = po.metadata['Last-Translator']
contributors_list.append(last_translator[:last_translator.find('<') - 1])
# Now we want to only show each contributor once, and to list them by how many
# contributions they have made - a rough guide to the effort they have put in.
contributors_dict = {}
for author in contributors_list:
author_copy = copy.copy(author)
if author_copy in ('', '(no author)', 'FULL NAME'):
# Skip bad data.
continue
# The creator of this project should always appear first in the list - so
# don't add him to this list, but hard-code his name.
if author_copy in ('Justin Driscoll', 'justin.driscoll'):
continue
# Handle contributors who appear under multiple names.
if author_copy == 'richardbarran':
author_copy = 'Richard Barran'
if author_copy in contributors_dict:
contributors_dict[author_copy] += 1
else:
contributors_dict[author_copy] = 1
with codecs.open('CONTRIBUTORS.txt', 'w', encoding='utf8') as f:
f.write('Photologue is made possible by all the people who have contributed'
' to it. A non-exhaustive list follows:\n\n')
f.write('Justin Driscoll\n')
for i in sorted(contributors_dict, key=contributors_dict.get, reverse=True):
f.write(i + '\n')
# And commit the new contributors file.
subprocess.check_output(["git", "commit", "-m", "Updated the list of contributors.", "CONTRIBUTORS.txt"])
def create_pot_file (input_file, pot_file, verbose):
"""
Create a .pot file from a translatables set of translatable strings
Args:
translatables: a list of tuples, each of which contain the line number
at which a translatable string was found, plus the string
itself
pot_file : the filename of the .pot to create
verbose : whether to display progress info
"""
translatables = get_translatables_from_file(input_file)
# check if the file exists. If it does, open it, otherwise create a new file
new_file = not os.path.exists(pot_file)
if new_file:
pofile = polib.POFile()
pofile.metadata= {
"Project-Id-Version": "Budgie Welcome",
"Report-Msgid-Bugs-To ": "*****@*****.**",
"POT-Creation-Date": "2016-02-27 12:48+0100",
"PO-Revision-Date": "YEAR-MO-DA HO:MI+ZONE",
"Last-Translator": "FULL NAME <EMAIL@ADDRESS>",
"Language-Team:": "LANGUAGE <*****@*****.**>",
"MIME-Version": "1.0",
"Content-Type": "text/plain; charset=UTF-8",
"Content-Transfer-Encoding": "8bit",}
else:
pofile = polib.pofile(pot_file)
source_path, source_file = os.path.split(input_file)
for translatable in translatables:
entry = polib.POEntry(
msgid = translatable[1],
msgstr = "",
occurrences = [(source_file, "%s" %str(translatable[0]).strip("[]"))])
if new_file:
pofile.append (entry)
if verbose:
print ('Added "%s" to pot file' %translatable[1])
else:
# add the entry only if it isn't already present...
present = False
for en in pofile:
if en.msgid == entry.msgid:
present = True
break
if not present:
pofile.append(entry)
if verbose:
print ('Added "%s" to pot file' %translatable[1])
else:
if verbose:
print ('Did not add "%s" - already present.' %translatable[1])
if verbose:
print ('Saving pot file to %s' %pot_file)
pofile.save(pot_file)
if verbose:
print ("Saved")
def translate_file(in_file, in_po, out_file, verbose):
""" Translate a file from English to another language
For every translatable string in_file, replace the corresponding
text in the input file with the appropriate translation from the .po
file. When all translations are done, write the resulting html to the
output file
Args:
in_file: the original filename whose contents are in English within
‌ enclosures
in_po : the filename of the .po file to use for the translation
out_file : the filename that the translation gets written to
"""
# get the translatable strings from the input file
translatables = get_translatables_from_file(in_file, trim_strs=False)
# read the entire contents of the file
content = open(in_file, 'r').read()
# read the po file
po_file = polib.pofile(in_po)
# for every translatable string, try to find and replace it with the appropriate
# translated
for translatable in translatables:
#can we find the translatable text in the input file?
if translatable[1] in content:
# remove \n and redundant spaces between words from the translatable string, so that
# it will match the entry in the .po file
trim_text = translatable[1].replace("\n", "")
trim_text = " ".join(trim_text.split())
# iterate through the po entries looking for the one which matches the current translatable
for entry in po_file:
if (entry.msgid == trim_text):
if (entry.msgstr !=""):
# replace the English text with the tranlated text.
# NOTE: ALL occurrences of the English text are replaced throughout the entire
# file
content = content.replace("%s%s%s" %("‌", translatable[1], "‌"), entry.msgstr)
if verbose:
print ("Translated %s >> %s" %(trim_text, entry.msgstr))
else:
# there's no translation, so instead just remove the ‌ characters from the string
content = content.replace("%s%s%s" %("‌", translatable[1], "‌"), entry.msgid)
if verbose:
print("No translation for %s, so ‌ characters removed" %entry.msgid)
break
else:
if verbose:
print ("WARNING: translatable string %s not found" %translatable[1])
# Post actions for i18n folder - Use relative paths.
content = content.replace('css/', '../../css/')
content = content.replace('js/', '../../js/')
content = content.replace('img/', '../../img/')
outfile = codecs.open(out_file, 'w', encoding='utf-8')
outfile.write(content)
outfile.close()
def check_messages(filename, report_empty=False):
"""
Checks messages in `filename` in various ways:
* Translations must have the same slots as the English.
* Messages can't have astral characters in them.
If `report_empty` is True, will also report empty translation strings.
Returns the problems, a list of tuples. Each is a description, a msgid, and
then zero or more translations.
"""
problems = []
pomsgs = polib.pofile(filename)
for msg in pomsgs:
# Check for characters Javascript can't support.
# https://code.djangoproject.com/ticket/21725
if astral(msg.msgstr):
problems.append(("Non-BMP char", msg.msgid, msg.msgstr))
if is_format_message(msg):
# LONG_DATE_FORMAT, etc, have %s etc in them, and that's ok.
continue
if msg.msgid_plural:
# Plurals: two strings in, N strings out.
source = msg.msgid + " | " + msg.msgid_plural
translation = " | ".join(
v for k, v in sorted(msg.msgstr_plural.items()))
empty = any(not t.strip() for t in msg.msgstr_plural.values())
else:
# Singular: just one string in and one string out.
source = msg.msgid
translation = msg.msgstr
empty = not msg.msgstr.strip()
if empty:
if report_empty:
problems.append(("Empty translation", source))
else:
id_tags = tags_in_string(source)
tx_tags = tags_in_string(translation)
# Check if tags don't match
if id_tags != tx_tags:
id_has = u", ".join(u'"{}"'.format(t)
for t in id_tags - tx_tags)
tx_has = u", ".join(u'"{}"'.format(t)
for t in tx_tags - id_tags)
if id_has and tx_has:
diff = u"{} vs {}".format(id_has, tx_has)
elif id_has:
diff = u"{} missing".format(id_has)
else:
diff = u"{} added".format(tx_has)
problems.append(("Different tags in source and translation",
source, translation, diff))
return problems
from __future__ import absolute_import, division, print_function, unicode_literals
import sys
import os
import json
import polib
LOGDEBUG = 'Debug'
LOGERROR = 'Error'
LOGNOTICE = 'Notice'
INFO_LABELS = {
'System.BuildVersion': '18.2',
}
PO = polib.pofile('resources/language/resource.language.en_gb/strings.po')
# Use the global_settings file
try:
with open('test/userdata/global_settings.json') as f:
GLOBAL_SETTINGS = json.load(f)
except OSError as e:
print("Error using 'test/userdata/global_settings.json' : %s" % e,
file=sys.stderr)
GLOBAL_SETTINGS = {
'locale.language': 'resource.language.en_gb',
'network.bandwidth': 0,
}
def executebuiltin(function, wait=False): # pylint: disable=unused-argument
if n < 4:
continue
f.write(line)
f.close()
print 'Complete'
# Compile language .po files to .mo
print 'Compiling PO files to MO...'
for po_file in os.listdir('languages'):
if not po_file.endswith('.po'):
continue
po_path = os.path.join('languages', po_file)
print 'Converting', po_path
po = polib.pofile(po_path, encoding='utf-8')
mo_path = po_path[:-3] + '.mo'
po.save_as_mofile(mo_path)
print 'Complete'
print
print 'Creating distribution package...'
# Create release dir and move release files inside it
os.mkdir(name)
# Copy files
for p_file in REL_FILES:
shutil.copy(p_file, os.path.join(name, p_file))
# Copy dirs
for p_dir in REL_DIRS:
shutil.copytree(p_dir, os.path.join(name, p_dir))
def clean_metadata(file):
"""
Clean up redundancies in the metadata caused by merging.
This reads in a PO file and simply saves it back out again.
"""
pofile(file).save()
data = yaml.safe_load(f)
for rule_str, production in data["rules"].items():
yield rule_str, production
productions = set()
rule_strs_by_production = defaultdict(list)
for filename in ("urdubiometer/settings/long.yml", "urdubiometer/settings/short.yml"):
for rule_str, production in get_rules(filename):
productions.add(production)
rule_strs_by_production[production].append(rule_str)
# load source translation file
po = polib.pofile("translations/urdubiometer.messages/en.po")
# check that each tag has required "<tag>__SHORT_DESCRIPTION"
for _ in sorted(productions):
for msgid in ("{}__SHORT_DESCRIPTION", "{}__WITH_VALUES_DESCRIPTION"):
msgid = msgid.format(_)
if not po.find(msgid):
comment = " | ".join([r for r in rule_strs_by_production[_]])
print("Adding", msgid)
po.append(polib.POEntry(msgid=msgid, msgstr="", comment=comment))
for _ in po:
msgid = _.msgid
m = re.match("^(.+)(__SHORT_DESCRIPTION|__WITH_VALUES_DESCRIPTION)", msgid)
if m:
Checks if the original string and the translation both end with dot or
not. """
parser = optparse.OptionParser(usage=USAGE)
(options, args) = parser.parse_args()
# Error checking (the pattern must be specified)
if len(args) < 1:
print >> sys.stderr, 'ERROR: Not enough arguments, the PO file must be specified'
sys.exit(1)
pofile = args[0]
print "Reading PO file: {0}".format(pofile)
po = polib.pofile(pofile)
for entry in po:
if entry.msgid.strip().endswith("..."):
if not (entry.msgstr.strip().endswith("...")
or entry.msgstr.strip().endswith("…".decode("utf-8"))):
print_error(entry)
elif entry.msgid.strip().endswith(
".") and not entry.msgstr.strip().endswith("."):
print_error(entry)
# elif not entry.msgid.strip().endswith(".") and entry.msgstr.strip().endswith("."):
# print_error(entry)
elif entry.msgid.strip().endswith(
":") and not entry.msgstr.strip().endswith(":"):
print_error(entry)
elif not entry.msgid.strip().endswith(":") and entry.msgstr.strip(
optparser.add_option(
"",
"--doctest",
help="run doctest of this tool, instead of check",
action="store_true",
)
(options, args) = optparser.parse_args()
if options.doctest:
import os
if 'TERM' in os.environ:
del os.environ['TERM']
import doctest
failures, tests = doctest.testmod()
sys.exit(failures and 1 or 0)
detected = []
warning = options.warning
for f in args:
detected.extend(
(f, pe, errors)
for pe, errors in check(polib.pofile(f), warning=warning))
if detected:
for f, pe, errors in detected:
for level, checker, error in errors:
sys.stderr.write('%s:%d:%s(%s): %s\n' %
(f, pe.linenum, level, checker, error))
sys.exit(1)
def set_new_translation(request):
"""
Post to include a new translation for a msgid
"""
if not get_user_can_translate(request.user):
return HttpResponseForbidden(
_('You have no permission to update translation catalogs'))
if not request.POST:
return HttpResponseBadRequest(
render_to_response('inlinetrans/response.html',
{'message': _('Invalid request method')},
context_instance=RequestContext(request)))
else:
result = {
'errors': True,
'question': False,
'message': _('Unknow error')
}
selected_pofile = None
msgid = smart_str(request.POST['msgid'])
msgstr = smart_str(request.POST['msgstr'])
retry = smart_str(request.POST['retry'])
lang = get_language()
# We try to update the catalog
if retry != 'false':
root_path = os.path.dirname(
os.path.normpath(
os.sys.modules[settings.SETTINGS_MODULE].__file__))
locale_path = os.path.dirname(
os.path.normpath(
os.sys.modules[settings.SETTINGS_MODULE].__file__))
makemessages(lang,
extensions=['.html'],
root_path=root_path,
locale_path=locale_path)
pos = find_pos(lang, include_djangos=True)
if pos:
for file_po in pos:
candidate = pofile(file_po)
poentry = candidate.find(msgid)
if poentry:
selected_pofile = candidate
poentry.msgstr = msgstr
if 'fuzzy' in poentry.flags:
poentry.flags.remove('fuzzy')
po_filename = file_po
break
# We can not find the msgid in any of the catalogs
if not selected_pofile:
result['message'] = _('"%(msgid)s" not found in any catalog' %
{'msgid': msgid})
if retry == 'false':
result['question'] = _(
'Do you want to update the catalog (this could take longer) and try again?'
)
return HttpResponse(simplejson.dumps(result),
mimetype='text/plain')
format_errors = validate_format(selected_pofile)
if format_errors:
result['message'] = format_errors
return HttpResponse(simplejson.dumps(result),
mimetype='text/plain')
if poentry and not format_errors:
try:
selected_pofile.metadata['Last-Translator'] = smart_str(
"%s %s <%s>" %
(request.user.first_name, request.user.last_name,
request.user.email))
selected_pofile.metadata['X-Translated-Using'] = smart_str(
"inlinetrans %s" % inlinetrans.get_version(False))
selected_pofile.metadata[
'PO-Revision-Date'] = datetime.datetime.now().strftime(
'%Y-%m-%d %H:%M%z')
except UnicodeDecodeError:
pass
selected_pofile.save()
selected_pofile.save_as_mofile(
po_filename.replace('.po', '.mo'))
result['errors'] = False
result['message'] = _('Catalog updated successfully')
elif not poentry:
result['message'] = _('PO entry not found')
return HttpResponse(simplejson.dumps(result), mimetype='text/plain')
if "top_srcdir" not in os.environ:
sys.stderr.write("$top_srcdir must be defined in the test environment\n")
sys.exit(99)
if "top_builddir" not in os.environ:
sys.stderr.write("$top_builddir must be defined in the test environment\n")
sys.exit(99)
# Update the .pot file with the latest strings
if os.system('make -C %s anaconda.pot-update' %
(os.environ['top_builddir'] + "/po")) != 0:
sys.stderr.write("Unable to update anaconda.pot")
sys.exit(1)
# Parse anaconda.pot and rearrange the POFile object into a dict of {msgid: POEntry}
pofile = polib.pofile(os.environ['top_srcdir'] + "/po/anaconda.pot")
msgs = {e.msgid: e for e in pofile}
# Look for each of the bad regexes
success = True
for badre in bad_strings.keys():
regex = re.compile(badre)
for msg in msgs.keys():
# Remove underscores to avoid trouble with underline-based accelerators
match = re.search(regex, msg.replace('_', ''))
if match:
# If this is something expected, decrement the occurrence count in expected_badness
badstr = match.group(0)
remainder = []
for occur in msgs[msg].occurrences:
if occur[0] in expected_badness and badstr in expected_badness[
print >> stream
program = os.path.basename(sys.argv[0])
print >> stream, __doc__ % {"program": program}
sys.exit(0)
if len(sys.argv) < 2:
usage(sys.stderr, "\nERROR: Not enough arguments")
filename = sys.argv[1]
debug = False
for i in range(1, len(sys.argv)):
arg = sys.argv.pop()
if arg == "--debug":
debug = True
po = polib.pofile(filename)
counter = 0
for entry in po:
counter += 1
match = patt.match(entry.comment)
if match:
default = match.group(1).replace('\n', ' ')
if "Default:" in default:
print "ERROR! There seems to be a duplicate Default entry for msgid '%s'" % entry.msgid
else:
if debug:
print "WARNING! No Default translation for msgid '%s'." % entry.msgid
sys.exit('Finished, checked all %d entries.' % counter)
#!/usr/bin/env python3
"""Append Welcome dialog strings to COOL UI pot file"""
import sys
import polib
welcome = polib.pofile(sys.argv[1],
autodetect_encoding=False,
encoding="utf-8",
wrapwidth=-1)
coolui = polib.pofile(sys.argv[2],
autodetect_encoding=False,
encoding="utf-8",
wrapwidth=78)
# Filter out unnecessary strings of meta tags from html2po output
for entry in welcome:
if 'html.head.meta' in entry.occurrences[0][0]:
continue
if entry.msgid == '':
continue
coolui.append(entry)
coolui.metadata['Content-Type'] = 'text/plain; charset=UTF-8'
coolui.save(sys.argv[2])
# require's polib from https://bitbucket.org/izi/polib/wiki/Home
#
# from top level of tree:
# check_translations.py /path/to/source/file
#
# Should output any untranslated or fuzzy lines from the file in a "lint" style
# NEEDS polib from http://pypi.python.org/pypi/polib
# or easy_install polib
import glob
import polib
#FIXME
PO_PATH = "po/"
po_files = glob.glob("%s/*.po" % PO_PATH)
for po_file in po_files:
print()
print(po_file)
p = polib.pofile(po_file)
for entry in p.untranslated_entries():
for line in entry.occurrences:
print("%s:%s" % (line[0], line[1]))
print("\t%s" % entry.msgid)
for entry in p.fuzzy_entries():
for line in entry.occurrences:
print("%s:%s" % (line[0], line[1]))
print("\t%s" % entry.msgid)
PLUGIN_LIVE_BRIDGE_PATH = "plugin://plugin.video.catchuptvandmore/resources/lib/main/"
M3U_ENTRY = '#EXTINF:-1 tvg-id="%s" tvg-logo="%s" group-title="%s",%s\n%s'
# arg0: tgv_id
# arg1: tgv_logo
# arg2: group_title
# arg3: label
# arg4: URL
# Parse english strings.po
# in order to recover labels
# Key format: "#30500"
# Value format: "France"
en_strings_po = {}
po = polib.pofile(EN_STRINGS_PO_FILEPATH)
for entry in po:
en_strings_po[entry.msgctxt] = entry.msgid
MANUAL_LABELS = {
'la_1ere': 'La 1ère',
'france3regions': 'France 3 Régions',
'euronews': 'Euronews',
'arte': 'Arte',
'dw': 'DW',
'france24': 'France 24',
'qvc': 'QVC',
'nhkworld': 'NHK World',
'cgtn': 'CGTN',
'paramountchannel': 'Paramount Channel',
'rt': 'RT',
def import_language(language):
''' Process the language.po file '''
return polib.pofile(
'resources/language/{language}/strings.po'.format(language=language))
def check_messages(filename, report_empty=False):
"""
Checks messages in various ways:
Translations must have the same slots as the English. Messages can't have astral
characters in them.
If report_empty is True, will also report empty translation strings.
"""
# Don't check English files.
if "/locale/en/" in filename:
return
# problems will be a list of tuples. Each is a description, and a msgid,
# and then zero or more translations.
problems = []
pomsgs = polib.pofile(filename)
for msg in pomsgs:
# Check for characters Javascript can't support.
# https://code.djangoproject.com/ticket/21725
if astral(msg.msgstr):
problems.append(("Non-BMP char", msg.msgid, msg.msgstr))
if msg.msgid_plural:
# Plurals: two strings in, N strings out.
source = msg.msgid + " | " + msg.msgid_plural
translation = " | ".join(v for k, v in sorted(msg.msgstr_plural.items()))
empty = any(not t.strip() for t in msg.msgstr_plural.values())
else:
# Singular: just one string in and one string out.
source = msg.msgid
translation = msg.msgstr
empty = not msg.msgstr.strip()
if empty:
if report_empty:
problems.append(("Empty translation", source))
else:
id_tags = tags_in_string(source)
tx_tags = tags_in_string(translation)
# Check if tags don't match
if id_tags != tx_tags:
id_has = u", ".join(u'"{}"'.format(t) for t in id_tags - tx_tags)
tx_has = u", ".join(u'"{}"'.format(t) for t in tx_tags - id_tags)
if id_has and tx_has:
diff = u"{} vs {}".format(id_has, tx_has)
elif id_has:
diff = u"{} missing".format(id_has)
else:
diff = u"{} added".format(tx_has)
problems.append((
"Different tags in source and translation",
source,
translation,
diff
))
if problems:
problem_file = filename.replace(".po", ".prob")
id_filler = textwrap.TextWrapper(width=79, initial_indent=" msgid: ", subsequent_indent=" " * 9)
tx_filler = textwrap.TextWrapper(width=79, initial_indent=" -----> ", subsequent_indent=" " * 9)
with codecs.open(problem_file, "w", encoding="utf8") as prob_file:
for problem in problems:
desc, msgid = problem[:2]
prob_file.write(u"{}\n{}\n".format(desc, id_filler.fill(msgid)))
for translation in problem[2:]:
prob_file.write(u"{}\n".format(tx_filler.fill(translation)))
prob_file.write(u"\n")
log.error(" {0} problems in {1}, details in .prob file".format(len(problems), filename))
else:
log.info(" No problems found in {0}".format(filename))
def setUpClass(cls):
gettextutil.check_version()
pot_path = gettextutil.update_pot(PODIR, "quodlibet")
cls.pot = polib.pofile(pot_path)
def __init__(self):
usage = """
Usage:
cinnamon-json-makepot -i | -r | [-js] <potfile name>
-js, --js - Runs xgettext on any javascript files in your directory before
scanning the settings-schema.json file. This allows you to generate
a .pot file for your entire applet at once.
***
The following two options should only be run in your applet, desklet, or
extension's directory
***
-i, --install - Compiles and installs any .po files contained in a po folder
to the system locale store. Use this option to test your translations
locally before uploading to Spices. It will use the applet, desklet,
or extension UUID as the translation domain
-r, --remove - The opposite of install, removes translations from the store.
Again, it uses the UUID to find the correct files to remove
<potfile name> - name of the .pot file to work with. This can be pre-existing,
or the name of a new file to use. If you leave off the .pot extension, it will
be automatically appended to the file name.
For instance:
cinnamon-json-makepot myapplet
Will generate a file called myapplet.pot, or append
to a file of that name. This can then be used by translators to be
made into a po file.
For example:
msginit --locale=fr --input=myapplet.pot
Will create "fr.po" for the French language. A translator can use a utility
such as poedit to add translations to this file, or edit the file manually.
.po files can be added to a "po" folder in your applet's directory,
and will be compiled and installed into the system when the applet is installed
via Cinnamon Settings.
"""
parser = OptionParser(usage=usage)
parser.add_option("-j",
"--js",
action="store_true",
dest="js",
default=False)
parser.add_option("-i",
"--install",
action="store_true",
dest="install",
default=False)
parser.add_option("-r",
"--remove",
action="store_true",
dest="remove",
default=False)
(options, args) = parser.parse_args()
if options.install:
self.do_install()
if options.remove:
self.do_remove()
if not args:
parser.print_help()
quit()
self.potname = args[0]
if not self.potname.endswith(".pot"):
self.potname = self.potname + ".pot"
self.domain = self.potname.replace(".pot", "")
self.potpath = os.path.join(os.getcwd(), self.potname)
if options.js:
try:
import subprocess
subprocess.call(["xgettext", "--version"])
except OSError:
print "xgettext not found, you may need to install the gettext package"
quit()
print " "
print "Running xgettext on JavaScript files..."
os.system("xgettext --language=C --keyword=_ --output=%s *.js" %
(self.potname))
self.current_parent_dir = ""
append = False
if os.path.exists(self.potpath):
append = True
if append:
self.po = polib.pofile(self.potpath)
else:
self.po = polib.POFile()
print "Scanning metadata.json and settings-schema.json..."
self.scan_dirs()
if append:
self.po.save()
else:
self.po.save(fpath=self.potpath)
print "Extraction complete"
quit()
translationCache = {}
# The purpose of this loop is to go to the podir scanning for PO files for each locale name
# Once we've found a PO file, we use PO lib to read every translated entry
# Using this, for each each language, we store a dict of entries - { nsilabel (comment) : translation (msgstr) }
# For untranslated entries, we use msgid instead of msgstr (i.e. default English string)
for root,dirs,files in os.walk(options.podir):
for file in files:
filename,ext = os.path.splitext(file)
if ext == ".po":
# Valid locale filename (fr.po, de.po etc)?
if filename in localeToName:
language = localeToName[filename]
translationCache[language] = collections.OrderedDict()
po = polib.pofile(os.path.join(root,file))
for entry in po.translated_entries():
# Loop through all our labels and add translation (each translation may have multiple labels)
for label in entry.comment.split():
translationCache[language][label] = escapeNSIS(entry.msgstr)
# For untranslated strings, let's add the English entry
for entry in po.untranslated_entries():
for label in entry.comment.split():
print("Warning: Label '%s' for language '%s' remains untranslated"%(label,language))
translationCache[language][label] = escapeNSIS(entry.msgid)
def tostr(obj):
if type(obj) == unicode:
return obj.encode("utf-8")
else:
return obj
import sys
import polib
import langdetect
#import goslate
from google.cloud import translate_v2 as translate
import time
from urllib.error import HTTPError
#gs = goslate.Goslate()
translate_client = translate.Client()
podata = polib.pofile(sys.argv[1])
podata.metadata['PO-Revision-Date'] = time.strftime("%Y-%m-%d %H:%M%z")
for entry in podata:
langs = {}
for l in langdetect.detect_langs(entry.msgid):
langs[l.lang] = l.prob
print(entry.msgid, langs)
if entry.msgstr == '' and not langs.get('ja'):
try:
#entry.msgstr = gs.translate(entry.msgid, 'ja')
entry.msgstr = translate_client.translate(
entry.msgid, target_language='ja')['translatedText']
except HTTPError as e:
print(e)
break
print(entry.msgstr)
#time.sleep(3)
def translate(self, src_file, src_lang, dest_file, dest_lang):
pofile = polib.pofile(src_file)
entries = []
ids = []
tokens = dict()
lastnum = 0
dcregex = re.compile('Default: "([^"]*)"', re.DOTALL)
for entry in pofile:
if entry.translated():
continue
entries.append(entry)
# Sorry folks, this will get squashed
txt = entry.msgid.replace('_', '')
if entry.comment and entry.comment.startswith('Default: '):
dcmatches = dcregex.findall(entry.comment)
if dcmatches and len(dcmatches) > 0:
txt = dcmatches[0]
if len(txt) > 500:
#We must skip entries longer than this, because google translate complains for these?
continue
tokenlist = re.findall('(\$\{[^\}]+\})', txt)
if tokenlist and len(tokenlist) > 0:
#There are $tokens in the msgid string.We will put this into our tokens dict and
#replace them with (numeric ids) so that when they come back translated we can replace the $tokens back
for token in tokenlist:
lastnum = lastnum + 1
id = "GT" + str(lastnum)
tokens[id] = token
txt = txt.replace(token, id)
ids.append(txt)
if len(ids) > self.STRINGS_PER_REQUEST:
print len(ids), ids
try:
strs = self.api.translate_many(ids, src_lang, dest_lang)
except Exception as inst:
print "Caught Exception: %s" % inst
strs = []
for id in ids:
try:
print "Translating: %s" % id
ret = self.api.translate(id, src_lang, dest_lang)
strs.append(ret)
except Exception as inst2:
strs.append('')
print "Caught Exception %s" % inst2
strs_replaced = []
for astr in strs:
ttokens = re.findall('(GT\d*)', astr)
if ttokens and len(ttokens) > 0:
for ttoken in ttokens:
print "Finding %s" % ttoken
if ttoken in tokens:
astr = astr.replace(ttoken, tokens[ttoken])
strs_replaced.append(astr)
assert len(ids) == len(strs_replaced) == len(entries)
for msg_str, e in zip(strs_replaced, entries):
e.msgstr = msg_str.strip()
ids = []
entries = []
tokens = dict()
lastnum = 0
pofile.save(dest_file)
