def _create_po_template(self, pot_path):
# type: (str) -> None
po_template = polib.POFile()
html_parser = etree.HTMLParser()
for html_path in self.files:
with open(html_path, 'rb') as html_file:
tree = etree.parse(html_file, html_parser)
for element in tree.iter():
if 'data-i18n' in element.keys():
new_entry = polib.POEntry(
msgid=element.get('data-i18n'),
occurrences=[(os.path.basename(html_path), element.sourceline)])
if new_entry not in po_template:
po_template.append(new_entry)
po_template.save(pot_path)
po_template.save(pot_path)
for html_path in self.files:
# Inline JavaScript may use underscorce function, e.g. univention/management/index.html
if tree.xpath('//script'):
message_catalogs.join_existing('JavaScript', pot_path, html_path)
def extract(self):
jsfile_rel = self.jsfile.replace(
translate_htmlconfig.get_sources_path(), '..')
with codecs.open(self.jsfile, 'r', 'utf-8') as fp:
linecount = 0
for line in fp.readlines():
linecount += 1
if line.startswith('var'):
var, message = line.split('=', 1)
var = var.split()[1]
message = message.strip()
message = message[1:-2]
entry = polib.POEntry(comment=var,
occurrences=[(jsfile_rel, linecount)
],
msgid=message,
msgstr=u'')
self.potfile.append(entry)
def po_2_pot(po_path, pot_path):
pofile = polib.pofile(po_path)
if not os.path.exists(pot_path):
# pot file doesn't exist, copy po
pofile.save(pot_path)
return
potfile = polib.pofile(pot_path)
for entry in pofile:
pot_entry = potfile.find(entry.msgid)
if pot_entry:
# update potfile msgstr to match pofile
pot_entry.msgstr = entry.msgstr
if not pot_entry:
# add new entry from pofile to potfile
pot_entry = polib.POEntry(msgid=entry.msgid,
msgstr=entry.msgstr,
occurrences=entry.occurrences)
potfile.append(pot_entry)
potfile.save(pot_path)
def _create_entry(self,
msgid,
msgstr="",
msgctxt=None,
comment=None,
tcomment=None):
"""
:param msgid: string, the entry msgid.
:param msgstr: string, the entry msgstr.
:param msgctxt: string, the entry context.
:param comment: string, the entry comment.
:param tcomment: string, the entry translator comment.
"""
entry = polib.POEntry(msgid=msgid,
msgstr=msgstr,
msgctxt=msgctxt,
comment=comment,
tcomment=tcomment)
self.file.append(entry)
def strings2pot(target, dest):
with open(target, "rt", encoding="utf-8") as fp:
contents = fp.read()
# We're reading an en.lproj file. We only care about the righthand part of the translation.
re_trans = re.compile(r'".*" = "(.*)";')
strings = re_trans.findall(contents)
if op.exists(dest):
po = polib.pofile(dest)
else:
po = polib.POFile()
for s in dedupe(strings):
s = unescape_cocoa_strings(s)
entry = po.find(s)
if entry is None:
entry = polib.POEntry(msgid=s)
po.append(entry)
# we don't know or care about a line number so we put 0
entry.occurrences.append((target, "0"))
entry.occurrences = dedupe(entry.occurrences)
po.save(dest)
def generate_translation_files(self):
for file_name in self.translations:
sheet_translations = self.translations[file_name]
po = polib.POFile()
po.check_for_duplicates = False
po.metadata = self.metadata
for translation in sheet_translations:
source = translation.key
if source:
entry = polib.POEntry(
msgid=translation.key,
msgstr=translation.translation or '',
occurrences=translation.occurrences,
msgctxt=translation.msgctxt
)
po.append(entry)
temp_file = tempfile.NamedTemporaryFile(delete=False)
po.save(temp_file.name)
self._generated_files.append(POFileInfo(file_name, temp_file.name))
return self._generated_files
def get_strings(self, source):
"""
Extracts translatable strings from the reporter:label attributes
in Evergreen's fieldmapper IDL file.
"""
self.pothead()
locator = xml.sax.xmlreader.Locator()
parser = xml.sax.make_parser()
handler = IDLHandler()
handler.setDocumentLocator(locator)
parser.setContentHandler(handler)
parser.parse(source)
for entity in handler.entities:
poe = polib.POEntry()
poe.occurrences = handler.entities[entity]
poe.msgid = entity
self.pot.append(poe)
self.idl = handler.entityized
def list2po(name, words, pof, comment=""):
entries = {}
if os.path.exists(pof):
for pe in open_po(pof):
entries[pe.msgid] = pe.msgstr
words = sorted(list(set(words)))
pofil = polib.POFile()
for word in words:
# skip A-100 type names
if len(word) < 7 and (word[1] == "-" or word[2] == "-"):
continue
args = {"msgid": word, "tcomment": comment}
if word in entries:
args["msgstr"] = entries[word]
pofil.append(polib.POEntry(**args))
pofil.header = header("data/" + name)
pofil.save(pof)
print "wrote " + os.path.relpath(
pof) + " (%d entries updated)" % len(entries)
def test_import_po_with_valid_string_clears_has_error_flag(self):
translation = StringTranslation.objects.create(
translation_of=String.objects.get(
data="This is some test content"),
context=TranslationContext.objects.get(path="test_charfield"),
locale=self.fr_locale,
data=
("This value is way too long for a char field so it should fail to publish and add an error to the translation. "
"This value is way too long for a char field so it should fail to publish and add an error to the translation. "
"This value is way too long for a char field so it should fail to publish and add an error to the translation."
),
has_error=True,
field_error=
"Ensure this value has at most 255 characters (it has 329).",
)
self.assertTrue(translation.has_error)
po = polib.POFile(wrapwidth=200)
po.metadata = {
"POT-Creation-Date": str(timezone.now()),
"MIME-Version": "1.0",
"Content-Type": "text/plain; charset=utf-8",
"X-WagtailLocalize-TranslationID": str(self.translation.uuid),
}
po.append(
polib.POEntry(
msgid="This is some test content",
msgctxt="test_charfield",
msgstr="Contenu de test",
))
warnings = self.translation.import_po(po)
self.assertEqual(warnings, [])
translation.refresh_from_db()
self.assertEqual(translation.context,
TranslationContext.objects.get(path="test_charfield"))
self.assertEqual(translation.locale, self.fr_locale)
self.assertEqual(translation.data, "Contenu de test")
self.assertFalse(translation.has_error)
def append_string(string, key, json, srcstring):
if type(string) == dict:
for k, v in string.items():
try:
append_string(v, k, string, srcstring[k])
except:
if 'en' in srcstring:
append_string(v, k, string, srcstring['en'])
else:
print "Error: string %s not in %s" % (k, path)
else:
if string:
lang = pathfile.basename()[0:2]
if lang not in pos:
po = Path() / 'exe' / 'locale' / lang / 'LC_MESSAGES' / 'exe.po'
pos[lang] = (polib.pofile(po), po)
entry = polib.POEntry(
msgid=srcstring,
msgstr=string
)
pos[lang][0].append(entry)
def test_import_po_with_invalid_translation_id(self):
po = polib.POFile(wrapwidth=200)
po.metadata = {
"POT-Creation-Date": str(timezone.now()),
"MIME-Version": "1.0",
"Content-Type": "text/plain; charset=utf-8",
"X-WagtailLocalize-TranslationID": "foo",
}
po.append(
polib.POEntry(
msgid="This is some test content",
msgctxt="test_charfield",
msgstr="Contenu de test",
))
warnings = self.translation.import_po(po)
self.assertEqual(warnings, [])
# Should delete both the translations
self.assertFalse(StringTranslation.objects.exists())
def build():
"""Build translation catalogs."""
metadata = {
'POT-Creation-Date': datetime.datetime.now(),
'PO-Revision-Date': datetime.datetime.now(),
'MIME-Version': '1.0',
'Content-Type': 'text/plain; charset=utf-8',
'Content-Transfer-Encoding': '8bit'
}
for lang in catalog.languages:
po = polib.POFile()
po.metadata = metadata
for entry in catalog.entries:
po.append(
polib.POEntry(msgid=entry['msgid'], msgstr=entry.get(lang,
'')))
path = os.path.join(LOCALE_DIR, lang, 'LC_MESSAGES')
if not os.path.isdir(path):
os.makedirs(path)
po.save(os.path.join(path, f'{NAMESPACE}.po'))
po.save_as_mofile(os.path.join(path, f'{NAMESPACE}.mo'))
def gen(domain, lang, input_file, input_dir):
if input_dir and input_file:
click.echo('duplication input:{} {}'.format(input_file, input_dir))
click.abort()
conf = json.load(open('{}/i18n.json'.format(os.getcwd())))
locale_dir = conf['locale_dir']
input_files = []
if input_file:
input_files.append(input_file)
if input_dir:
for root, _, files in os.walk(input_dir):
input_files.extend([os.path.join(root, f) for f in files if f.endswith('.py')])
po_msgs = []
for file in input_files:
extracted_msgs = extract(file, '_')
po_msgs.extend(extracted_msgs)
po_msgs = list(set(po_msgs))
po_entries = [polib.POEntry(msgid=msg, msgstr="") for msg in po_msgs]
p = po.gen(po_entries, **conf['metadata'])
po.save(p, domain, locale_dir, lang)
def terminal_quest_assets_to_pot(assets_dir, pot_dir):
pot = polib.POFile()
pot.metadata = {
'Project-Id-Version': 'terminal-quest',
'Report-Msgid-Bugs-To': '',
'POT-Creation-Date': time.strftime("%Y-%m-%d %H:%M%z"),
'PO-Revision-Date': time.strftime("%Y-%m-%d %H:%M%z"),
'Last-Translator': '',
'Language-Team': 'English',
'MIME-Version': '1.0',
'Content-Type': 'text/plain; charset=utf-8',
'Content-Transfer-Encoding': '8bit',
}
for f in os.listdir(assets_dir):
with open(os.path.join(assets_dir, f), 'r') as asset_file:
entry = polib.POEntry(msgid=asset_file.read(),
msgstr=u'',
occurrences=[(f, '')])
pot.append(entry)
pot.save(os.path.join(pot_dir, 'assets.pot'))
def _write_entries(po_files, languages, msgid, msgstrs, metadata, comment):
"""
Write msgstr for every language with all needed metadata and comment.
Metadata are parser from string into dict, so read them only from gdocs.
"""
start = re.compile(r'^[\s]+')
end = re.compile(r'[\s]+$')
for i, lang in enumerate(languages):
meta = ast.literal_eval(metadata)
entry = polib.POEntry(**meta)
entry.tcomment = comment
entry.msgid = msgid
if msgstrs[i]:
start_ws = start.search(msgid)
end_ws = end.search(msgid)
entry.msgstr = str(start_ws.group() if start_ws else '') + \
unicode(msgstrs[i].strip()) + \
str(end_ws.group() if end_ws else '')
else:
entry.msgstr = ''
po_files[lang].append(entry)
def dump_translation_memory(self, tm):
"""
Dump translation memory to a pot-file.
Sort messages in pot-file by location (if there's more locations, sort locations
alphabetiaclly first) so that translators can process canvases alphabetially and
easily review exported images in a folder one by one.
"""
container = []
for text, locations in tm.items():
container.append((sorted([l for l in locations]), text))
container.sort(key=lambda x: x[0][0])
pot = polib.POFile()
for locations, text in container:
entry = polib.POEntry(msgid=text,
msgstr=text,
occurrences=[(location, '0')
for location in locations])
pot.append(entry)
pot.save(os.path.splitext(self.args.source)[0] + '.pot')
def update_po_file(po_lib, t9n_list=None):
if t9n_list:
for t9n in t9n_list:
msgid = t9n.msgid
msgctxt = t9n.msgctxt
msgid_plural = t9n.plural
po_entry = None
if msgid:
if msgctxt:
po_entry = po_lib.find(msgid, msgctxt=msgctxt)
if not msgctxt and not po_entry:
# search for poentry with only a msgid if no msgctxt is available
po_entry = po_lib.find(msgid)
if not po_entry:
# no po entry found create a new one.
po_entry = polib.POEntry(msgid=msgid, msgctxt=msgctxt, msgid_plural=msgid_plural)
po_lib.append(po_entry)
po_lib.save()
def extract_loc_pots(path: Path):
pot_dir = CURRENT_DIR / "MTGA" / "t"
mtga_files = (i for i in os.listdir(path) if i.endswith(".mtga"))
with typer.progressbar(mtga_files,
label="Extracting Game Locale") as progress:
for filename in progress:
po = polib.POFile()
with open(path / filename) as infile:
data = json.load(infile)
for idx, obj in enumerate(data, start=1):
key = obj["key"]
bundle = obj["bundle"] or "Internal"
text = get_loc_en_trans(obj["translations"])
entry = polib.POEntry(
msgctxt=key,
msgid=text,
msgstr="",
)
po.append(entry)
po.save(f"{pot_dir}/MTGA_{bundle}.pot")
def generate_po(nodes, filename):
# Create po file
po = polib.POFile()
# Append titles & descriptions
string_set = set()
for node in nodes:
for key in ["title", "description"]:
value = node.get(key)
if not value or node.get("kind") == "Separator":
continue
if value in string_set:
continue
string_set.add(value)
entry = polib.POEntry(msgid=value,
msgstr="",
comment="%s %s" % (node["kind"], key))
po.append(entry)
# Save()
po.save(filename)
def makelocale(lang, plural_forms=None):
plural_forms = plural_forms or "nplurals=3; plural=(n==1) ? 0 : (n>=2 && n<=4) ? 1 : 2;"
import polib
po = polib.POFile()
tzdata = maketzdata()
countries = set()
regions = set()
cities = set()
l18n.set_language("en")
for luci_tz, country, city, zoneinfo in tzdata:
regions.add(luci_tz.split("/")[0])
countries.add(unicode(l18n.territories[country]))
cities.add(city)
po.metadata = {
'Project-Id-Version': "Foris TZ info translations",
'POT-Creation-Date': "2016-04-28 15:04+0200",
'PO-Revision-Date': "2016-04-28 13:55+0200",
'Last-Translator': "Automatically generated <*****@*****.**>",
'Language': lang,
'MIME-Version': "1.0",
'Content-Type': "text/plain; charset=UTF-8",
'Content-Transfer-Encoding': "8bit",
'Plural-Forms': plural_forms,
}
# Activate translation for the requested language
l18n.set_language(lang)
translations = {}
for part in (regions, cities, countries):
for x in part:
translations[x] = unicode(l18n.translation.L18NLazyString(x))
for msgid, msgstr in translations.iteritems():
po.append(polib.POEntry(msgid=msgid, msgstr=msgstr))
po.save("foris/locale/%s/LC_MESSAGES/tzinfo.po" % lang)
def do_filter(pofile, locale, conn):
do_filter = False
newpo = polib.POFile()
newpo.metadata['Project-Id-Version'] = '1.0'
newpo.metadata['Report-Msgid-Bugs-To'] = '*****@*****.**'
newpo.metadata['POT-Creation-Date'] = '2007-10-18 14:00+0100'
newpo.metadata['PO-Revision-Date'] = '2007-10-18 14:00+0100'
newpo.metadata['Last-Translator'] = 'you <*****@*****.**>'
newpo.metadata['Language-Team'] = 'RuXu WU<*****@*****.**>'
newpo.metadata['MIME-Version'] = '1.0'
newpo.metadata['Content-Type'] = 'text/plain; charset=utf-8'
newpo.metadata['Content-Transfer-Encoding'] = '8bit'
try:
po = polib.pofile(pofile)
except:
print 'load po file %s failure!' % (pofile)
c = conn.cursor()
for entry in po:
msgstr = get_from_db(locale, entry.msgid, c)
if msgstr == '':
do_filter = True
newentry = polib.POEntry('', '')
newentry.msgid = entry.msgid
newentry.occurrences = entry.occurrences
newpo.append(newentry)
c.close()
if do_filter:
if output[-1] == '/':
path = output + locale + '/LC_MESSAGES/'
else:
path = output + '/' + locale + '/LC_MESSAGES/'
if os.access(path, 0) == 0:
os.makedirs(path)
path += os.path.split(pofile)[1]
newpo.encoding = 'utf8'
newpo.save(path)
def write_languagefile(outputfilename, l10n_src, old_l10n_data):
outputfilename += '.po'
po = polib.POFile()
po.metadata = {
'Project-Id-Version': 'teeworlds-0.7_dev',
'Report-Msgid-Bugs-To': '*****@*****.**',
'POT-Creation-Date': time.strftime("%Y-%m-%d %H:%M%z"),
'PO-Revision-Date': time.strftime("%Y-%m-%d %H:%M%z"),
'Language-Team': 'Teeworlds Translations <*****@*****.**>',
'MIME-Version': '1.0',
'Content-Type': 'text/plain; charset=utf-8',
'Content-Transfer-Encoding': '8bit',
}
translations = {}
for type_ in (
JSON_KEY_OLDTRANSL,
JSON_KEY_UNTRANSL,
JSON_KEY_TRANSL,
):
translations.update({
t[JSON_KEY_OR]: t[JSON_KEY_TR]
for t in old_l10n_data[type_] if t[JSON_KEY_TR]
})
all_items = set(translations) | set(l10n_src)
tsl_items = set(translations) & set(l10n_src)
old_items = set(translations) - set(l10n_src)
new_items = set(l10n_src) - set(translations)
for msg in all_items:
po.append(
polib.POEntry(
msgid=msg,
msgstr=translations.get(msg, ""),
obsolete=(msg in old_items),
occurrences=l10n_src[msg],
))
po.save(outputfilename)
def testload(self):
"""
Load a translated PO file and compare to a generated one
"""
import basel10n
poload = basel10n.BaseL10N()
poload.loadpo(self.canonpo)
pogen = basel10n.BaseL10N()
pogen.pothead('Evergreen 1.4', '1999-12-31 23:59:59 -0400')
pogen.pot.metadata['PO-Revision-Date'] = '2007-12-08 23:14:20 -0400'
pogen.pot.metadata['Last-Translator'] = ' Dan Scott <*****@*****.**>'
pogen.pot.metadata['Language-Team'] = 'fr-CA <*****@*****.**>'
for msg in self.poentries:
poe = polib.POEntry()
for x in msg['occurrences']:
poe.occurrences.append((x['name'], x['line']))
poe.msgid = msg['msgid']
if msg.has_key('msgstr'):
poe.msgstr = msg['msgstr']
pogen.pot.append(poe)
self.assertEqual(unicode(poload), unicode(pogen))
def test_import_po_file_with_new_language(self):
self._import_po_entries('es', [
polib.POEntry(
occurrences=[('organizations.organization@name@one', '')],
msgid='organization_one_translation_name_msgid',
msgstr='organization_one_translation_name_msgstr_es')
])
master = self.translatable_objects[0]
translation = master.get_translation('es')
translation.refresh_from_db()
self.assertEqual(translation.name,
'organization_one_translation_name_msgstr_es')
self.assertEqual(translation.description, '')
master = self.translatable_objects[1]
with self.assertRaises(master.translations.model.DoesNotExist):
master.get_translation('es')
master = self.translatable_objects[2]
with self.assertRaises(master.translations.model.DoesNotExist):
master.get_translation('es')
def serialize(self):
po = polib.POFile()
for unit in self.units:
occurences = unit.occurrences[:]
entry = polib.POEntry(
msgid=unit.key,
msgstr=unit.value,
comment=getattr(unit, "comment", ""),
tcomment=getattr(unit, "translator_comment", ""),
occurences=occurences,
obsolete=unit.obsolete,
)
if unit.context:
entry.msgctxt = unit.context
if hasattr(unit, "po_flags"):
entry.flags = unit.po_flags[:]
if unit.state == State.UNFINISHED:
entry.flags.append("fuzzy")
po.append(entry)
return str(po)
def main():
args = sys.argv[1:]
tall = False
longstring = False
wrap = False
if "--tall" in args:
tall = True
if "--long" in args:
longstring = True
if "--wrap" in args:
wrap = True
potfile = PotFile()
for pot_entry in potfile.po:
entry = polib.POEntry(msgid=pot_entry.msgid,
msgstr=gen_msg(pot_entry.msgid, tall, longstring,
wrap),
occurrences=pot_entry.occurrences)
po.append(entry)
po.save("po/en_US.po")
def appendEntries(po, translations, oldValues={}):
(sortedKeys, translations) = translations
for key in sortedKeys:
positions = translations[key]
oldval = key
if key in oldValues:
oldval = oldValues[key]
occurences=[]
for position in positions:
(clazz, posInClass) = position
for pos in posInClass:
occurences.append((clazz, pos))
entry = polib.POEntry(
msgid=key,
msgstr=oldval,
occurrences=occurences
)
po.append(entry)
def _save_current_msgid(self, msgstr='', fuzzy=False):
# raise 'msgid' event
if raise_skip_event(
self.events,
'msgid',
self,
self.current_msgid,
msgstr,
self.current_msgctxt,
self.current_tcomment,
['fuzzy'] if fuzzy else [],
):
return
if self.current_msgid:
if (not self.disable_next_block and not self.disable) or \
self.enable_next_block:
self._save_msgid(
self.current_msgid,
msgstr=msgstr or self.msgstr,
msgctxt=self.current_msgctxt,
tcomment=self.current_tcomment,
fuzzy=fuzzy,
)
else:
self.disabled_entries.append(
polib.POEntry(
msgid=self.current_msgid,
msgstr=msgstr or self.msgstr,
msgctxt=self.current_msgctxt,
tcomment=self.current_tcomment,
flags=['fuzzy'] if fuzzy else [],
), )
self.disable_next_block = False
self.enable_next_block = False
self.current_msgid = ''
self.current_tcomment = None
self.current_msgctxt = None
def convert(self):
pofile = polib.POFile()
pofile.metadata = {
'Project-Id-Version': '1.0',
'Report-Msgid-Bugs-To': 'none',
'POT-Creation-Date': '2007-10-18 14:00+0100',
'PO-Revision-Date': '2007-10-18 14:00+0100',
'Last-Translator': '*****@*****.**',
'Language-Team': 'Catalan <*****@*****.**>',
'MIME-Version': '1.0',
'Content-Type': 'text/plain; charset=utf-8',
'Content-Transfer-Encoding': '8bit',
'Plural-Forms': 'nplurals=2; plural=n != 1;',
}
entries = 0
sources = self._read_source()
targets = self._read_target()
saved = set()
for key, source in sources:
if key in targets:
target = targets[key]
else:
target = ""
if source in saved:
continue
if len(source) == 0:
continue
entry = polib.POEntry(msgid=source, msgstr=target, comment=key)
pofile.append(entry)
entries = entries + 1
saved.add(source)
pofile.save(self.output_file)
def clonePOEntry(src):
tgt = polib.POEntry()
tgt.msgid = src.msgid
tgt.msgstr = src.msgstr
tgt.msgid_plural = src.msgid_plural
if src.msgstr_plural:
tgt.msgstr_plural = src.msgstr_plural
tgt.msgctxt = src.msgctxt
tgt.obsolete = src.obsolete
tgt.encoding = src.encoding
tgt.msgid = src.msgid
tgt.msgstr = src.msgstr
tgt.comment = src.comment
tgt.tcomment = src.tcomment
if src.occurrences:
tgt.occurrences = src.occurrences[:]
if src.flags:
tgt.flags = src.flags[:]
tgt.previous_msgctxt = src.previous_msgctxt
tgt.previous_msgid = src.previous_msgid
tgt.previous_msgid_plural = src.previous_msgid_plural
tgt.linenum = src.linenum
return tgt
