def _build(self, locale, messages, path, pathGlobal=None):
'''
Builds a catalog based on the provided locale paths, the path is used as the main source any messages that are not
found in path locale but are part of messages will attempt to be extracted from the global path locale.
@param locale: Locale
The locale.
@param messages: Iterable(Message)
The messages to build the PO file on.
@param path: string
The path of the targeted PO file from the locale repository.
@param pathGlobal: string|None
The path of the global PO file from the locale repository.
@return: file like object
File like object that contains the PO file content
'''
assert isinstance(locale, Locale), 'Invalid locale %s' % locale
assert isinstance(messages, Iterable), 'Invalid messages %s' % messages
assert isinstance(path, str), 'Invalid path %s' % path
assert pathGlobal is None or isinstance(pathGlobal, str), 'Invalid global path %s' % pathGlobal
if isfile(path):
with open(path) as fObj: catalog = read_po(fObj, locale)
else:
catalog = Catalog(locale, creation_date=datetime.now(), **self.catalog_config)
if pathGlobal and isfile(pathGlobal):
with open(pathGlobal) as fObj: catalogGlobal = read_po(fObj, locale)
else:
catalogGlobal = None
self._processCatalog(catalog, messages, fallBack=catalogGlobal)
catalog.revision_date = datetime.now()
return catalog
def test_update(self):
template = Catalog()
template.add("1")
template.add("2")
template.add("3")
tmpl_file = os.path.join(self._i18n_dir(), "temp-template.pot")
with open(tmpl_file, "wb") as outfp:
write_po(outfp, template)
po_file = os.path.join(self._i18n_dir(), "temp1.po")
self.cli.run(sys.argv + ["init", "-l", "fi", "-o", po_file, "-i", tmpl_file])
with open(po_file, "r") as infp:
catalog = read_po(infp)
assert len(catalog) == 3
# Add another entry to the template
template.add("4")
with open(tmpl_file, "wb") as outfp:
write_po(outfp, template)
self.cli.run(sys.argv + ["update", "-l", "fi_FI", "-o", po_file, "-i", tmpl_file])
with open(po_file, "r") as infp:
catalog = read_po(infp)
assert len(catalog) == 4 # Catalog was updated
def run(self, root):
i18n_dir = self.extension.getConfig('i18n_dir')
pot_path = os.path.join(i18n_dir, 'messages.pot')
if os.path.exists(pot_path):
with open(pot_path, 'r') as f:
catalog = pofile.read_po(f)
else:
catalog = Catalog()
lang = self.extension.getConfig('i18n_lang')
mo_path = os.path.join(i18n_dir, lang, 'LC_MESSAGES', 'messages.mo')
po_path = os.path.join(i18n_dir, lang, 'LC_MESSAGES', 'messages.po')
if os.path.exists(po_path):
with open(po_path, 'r') as f:
lang_catalog = pofile.read_po(f)
with open(mo_path, 'w') as mo:
mofile.write_mo(mo, lang_catalog)
translations = Translations.load(i18n_dir, locales=[lang])
self.translate(catalog, translations, root)
with open(pot_path, 'w') as pot_file:
pofile.write_po(pot_file, catalog)
def test_update(self):
template = Catalog()
template.add("1")
template.add("2")
template.add("3")
tmpl_file = os.path.join(self._i18n_dir(), 'temp-template.pot')
with open(tmpl_file, "wb") as outfp:
write_po(outfp, template)
po_file = os.path.join(self._i18n_dir(), 'temp1.po')
self.cli.run(sys.argv + ['init',
'-l', 'fi',
'-o', po_file,
'-i', tmpl_file
])
with open(po_file, "r") as infp:
catalog = read_po(infp)
assert len(catalog) == 3
# Add another entry to the template
template.add("4")
with open(tmpl_file, "wb") as outfp:
write_po(outfp, template)
self.cli.run(sys.argv + ['update',
'-l', 'fi_FI',
'-o', po_file,
'-i', tmpl_file])
with open(po_file, "r") as infp:
catalog = read_po(infp)
assert len(catalog) == 4 # Catalog was updated
def import_file(self, locale, po_path):
if locale is None:
raise Error('Must specify locale.')
if not os.path.exists(po_path):
raise Error('Couldn\'t find PO file: {}'.format(po_path))
babel_locale = external_to_babel_locales.get(locale, locale)
pod_translations_dir = os.path.join(
'translations', babel_locale, 'LC_MESSAGES')
pod_po_path = os.path.join(pod_translations_dir, 'messages.po')
if self.pod.file_exists(pod_po_path):
existing_po_file = self.pod.open_file(pod_po_path)
existing_catalog = pofile.read_po(existing_po_file, babel_locale)
po_file_to_merge = open(po_path)
catalog_to_merge = pofile.read_po(po_file_to_merge, babel_locale)
for message in catalog_to_merge:
if message.id not in existing_catalog:
existing_catalog[message.id] = message
else:
existing_catalog[message.id].string = message.string
existing_po_file = self.pod.open_file(pod_po_path, mode='w')
pofile.write_po(existing_po_file, existing_catalog, width=80,
omit_header=True, sort_output=True, sort_by_file=True)
text = 'Imported {} translations: {}'
self.pod.logger.info(text.format(len(catalog_to_merge), babel_locale))
else:
abs_po_path = self.pod.abs_path(pod_po_path)
abs_po_dir = os.path.dirname(abs_po_path)
_mkdir(abs_po_dir)
shutil.copyfile(po_path, abs_po_path)
self.pod.logger.info('Imported new catalog: {}'.format(babel_locale))
def merge_file(source_filename, target_filename, template_filename):
if source_filename.endswith('.po'):
merge(pofile.read_po(open(source_filename)), target_filename)
elif source_filename.endswith('.xml'):
handler = XmbCatalogReader(pofile.read_po(open(template_filename)))
xml.sax.parse(open(source_filename), handler)
merge(handler.catalog, target_filename)
def run(self):
if tx_executable is None:
sys.exit('Transifex client executable (tx) not found.')
from babel.messages import pofile
if not self.skip_pull:
txpull_cmd = [
tx_executable,
'pull',
'--force',
'--resource=musicbrainz.attributes,musicbrainz.countries',
'--source',
'--language=none',
]
self.spawn(txpull_cmd)
countries = dict()
countries_potfile = os.path.join('po', 'countries', 'countries.pot')
isocode_comment = u'iso.code:'
with open(countries_potfile, 'rb') as f:
log.info('Parsing %s' % countries_potfile)
po = pofile.read_po(f)
for message in po:
if not message.id or not isinstance(message.id, unicode):
continue
for comment in message.auto_comments:
if comment.startswith(isocode_comment):
code = comment.replace(isocode_comment, u'')
countries[code] = message.id
if countries:
self.countries_py_file(countries)
else:
sys.exit('Failed to extract any country code/name !')
attributes = dict()
attributes_potfile = os.path.join('po', 'attributes', 'attributes.pot')
extract_attributes = (
u'DB:cover_art_archive.art_type/name',
u'DB:medium_format/name',
u'DB:release_group_primary_type/name',
u'DB:release_group_secondary_type/name',
)
with open(attributes_potfile, 'rb') as f:
log.info('Parsing %s' % attributes_potfile)
po = pofile.read_po(f)
for message in po:
if not message.id or not isinstance(message.id, unicode):
continue
for loc, pos in message.locations:
if loc in extract_attributes:
attributes[u"%s:%03d" % (loc, pos)] = message.id
if attributes:
self.attributes_py_file(attributes)
else:
sys.exit('Failed to extract any attribute !')
def test_abort_invalid_po_file(self):
invalid_po = '''
msgctxt ""
"{\"checksum\": 2148532640, \"cxt\": \"collector_thankyou\", \"id\": "
"270005359}"
msgid ""
"Thank you very much for your time.\n"
"If you have any questions regarding this survey, please contact Fulano "
"at [email protected]"
msgstr "Merci de prendre le temps de remplir le sondage.
Pour toute question, veuillez communiquer avec Fulano à [email protected]
"
'''
invalid_po_2 = '''
msgctxt ""
"{\"checksum\": 2148532640, \"cxt\": \"collector_thankyou\", \"id\": "
"270005359}"
msgid ""
"Thank you very much for your time.\n"
"If you have any questions regarding this survey, please contact Fulano "
"at [email protected]."
msgstr "Merci de prendre le temps de remplir le sondage.
Pour toute question, veuillez communiquer avec Fulano a [email protected]
"
'''
# Catalog not created, throws Unicode Error
buf = StringIO(invalid_po)
output = None
# This should only be thrown under py27
if sys.version_info.major == 2:
with self.assertRaises(UnicodeEncodeError):
output = pofile.read_po(buf, locale='fr', abort_invalid=False)
assert not output
else:
output = pofile.read_po(buf, locale='fr', abort_invalid=False)
assert isinstance(output, Catalog)
# Catalog not created, throws PoFileError
buf = StringIO(invalid_po_2)
output = None
with self.assertRaises(pofile.PoFileError) as e:
output = pofile.read_po(buf, locale='fr', abort_invalid=True)
assert not output
# Catalog is created with warning, no abort
buf = StringIO(invalid_po_2)
output = pofile.read_po(buf, locale='fr', abort_invalid=False)
assert isinstance(output, Catalog)
# Catalog not created, aborted with PoFileError
buf = StringIO(invalid_po_2)
output = None
with self.assertRaises(pofile.PoFileError) as e:
output = pofile.read_po(buf, locale='fr', abort_invalid=True)
assert not output
def test_locale_gets_overridden_by_file(self):
buf = StringIO(r'''
msgid ""
msgstr ""
"Language: en_US\n"''')
catalog = pofile.read_po(buf, locale='de')
self.assertEqual(Locale('en', 'US'), catalog.locale)
buf = StringIO(r'''
msgid ""
msgstr ""
"Language: ko-KR\n"''')
catalog = pofile.read_po(buf, locale='de')
self.assertEqual(Locale('ko', 'KR'), catalog.locale)
def test_overwrite_po_success(minimal_i18n_settings, temp_builds_dir,
fixtures_settings):
"""
safe_write_po usage for overwritting file
"""
basepath = temp_builds_dir.join('i18n_overwrite_po_success')
# Copy sample project to temporary dir
samplename = 'minimal_i18n'
samplepath = os.path.join(fixtures_settings.fixtures_path, samplename)
destination = os.path.join(basepath.strpath, samplename)
shutil.copytree(samplepath, destination)
dummy_name = "dummy_pot.pot"
dummy_pot = os.path.join(destination, dummy_name)
# Get manager with settings
settings = minimal_i18n_settings(destination)
manager = I18NManager(settings)
# Create a dummy catalog to write
catalog = Catalog(header_comment="# Foobar")
catalog.add('foo %(name)s', locations=[('main.py', 1)], flags=('fuzzy',))
# Write it
manager.safe_write_po(catalog, dummy_pot)
# Check it
with io.open(dummy_pot, 'rb') as f:
dummy_catalog = read_po(f)
assert dummy_catalog.header_comment == "# Foobar"
# New dummy catalog to overwrite previous one
catalog = Catalog(header_comment="# Zob")
catalog.add('ping', string='pong', locations=[('nope.py', 42)])
# Write it
manager.safe_write_po(catalog, dummy_pot)
# List existing pot file at root
pots = [item for item in os.listdir(destination) if item.endswith('.pot')]
# No other pot file
assert pots == [dummy_name]
# Check it again
with io.open(dummy_pot, 'rb') as f:
dummy_catalog = read_po(f)
assert dummy_catalog.header_comment == "# Zob"
assert dummy_catalog["ping"] == Message('ping', string='pong')
def write_po_files(domain):
for locale in GETTEXT_LANGUAGES_LOCALE:
popath = os.path.join("locale", locale, "LC_MESSAGES", domain + ".po")
potpath = os.path.join("locale", domain + ".pot")
with open(popath, 'r') as po_f, open(potpath, 'r') as pot_f:
template = read_po(pot_f)
catalog = read_po(po_f)
catalog.update(template)
with open(popath, 'w') as po_f:
write_po(po_f, catalog, ignore_obsolete=True)
potpath = os.path.join("locale", domain + ".pot")
os.unlink(potpath)
def load_po(filename):
"""read po/pot file and return catalog object
:param unicode filename: path to po/pot file
:return: catalog object
"""
# pre-read to get charset
with io.open(filename, 'rb') as f:
cat = pofile.read_po(f)
charset = cat.charset or 'utf-8'
# To decode lines by babel, read po file as binary mode and specify charset for
# read_po function.
with io.open(filename, 'rb') as f: # FIXME: encoding VS charset
return pofile.read_po(f, charset=charset)
def test_overwrite_po_fail(minimal_i18n_settings, temp_builds_dir,
fixtures_settings):
"""
safe_write_po usage for overwritting file failing but left untouched
initial file
"""
basepath = temp_builds_dir.join('i18n_overwrite_po_fail')
# Copy sample project to temporary dir
samplename = 'minimal_i18n'
samplepath = os.path.join(fixtures_settings.fixtures_path, samplename)
destination = os.path.join(basepath.strpath, samplename)
shutil.copytree(samplepath, destination)
dummy_name = "dummy_pot.pot"
dummy_pot = os.path.join(destination, dummy_name)
# Get manager with settings
settings = minimal_i18n_settings(destination)
manager = I18NManager(settings)
# Create a dummy catalog to write
catalog = Catalog(header_comment="# Foobar")
catalog.add('foo %(name)s', locations=[('main.py', 1)], flags=('fuzzy',))
# Write it
manager.safe_write_po(catalog, dummy_pot)
# Check it
with io.open(dummy_pot, 'rb') as f:
dummy_catalog = read_po(f)
assert dummy_catalog.header_comment == "# Foobar"
# Try to overwrite with empty catalog to raise error
with pytest.raises(TypeError):
manager.safe_write_po(None, dummy_pot)
# List existing pot file at root
pots = [item for item in os.listdir(destination) if item.endswith('.pot')]
# No other pot file
assert pots == [dummy_name]
# Check it again
with io.open(dummy_pot, 'rb') as f:
dummy_catalog = read_po(f)
# Initial has been left untouched
assert dummy_catalog.header_comment == "# Foobar"
def test_header_entry(self):
buf = StringIO(r'''\
# SOME DESCRIPTIVE TITLE.
# Copyright (C) 2007 THE PACKAGE'S COPYRIGHT HOLDER
# This file is distributed under the same license as the PACKAGE package.
# FIRST AUTHOR <EMAIL@ADDRESS>, 2007.
#
#, fuzzy
msgid ""
msgstr ""
"Project-Id-Version: 3.15\n"
"Report-Msgid-Bugs-To: Fliegender Zirkus <*****@*****.**>\n"
"POT-Creation-Date: 2007-09-27 11:19+0700\n"
"PO-Revision-Date: 2007-09-27 21:42-0700\n"
"Last-Translator: John <*****@*****.**>\n"
"Language-Team: German Lang <*****@*****.**>\n"
"Plural-Forms: nplurals=2; plural=(n != 1)\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=iso-8859-2\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: Babel 1.0dev-r313\n"
''')
catalog = pofile.read_po(buf)
self.assertEqual(1, len(list(catalog)))
self.assertEqual(u'3.15', catalog.version)
self.assertEqual(u'Fliegender Zirkus <*****@*****.**>',
catalog.msgid_bugs_address)
self.assertEqual(datetime(2007, 9, 27, 11, 19,
tzinfo=FixedOffsetTimezone(7 * 60)),
catalog.creation_date)
self.assertEqual(u'John <*****@*****.**>', catalog.last_translator)
self.assertEqual(u'German Lang <*****@*****.**>', catalog.language_team)
self.assertEqual(u'iso-8859-2', catalog.charset)
self.assertEqual(True, list(catalog)[0].fuzzy)
def machine_translate(self):
locale = str(self.locale)
domain = 'messages'
infile = self.pod.open_file(self.pod_path, 'U')
try:
babel_catalog = pofile.read_po(infile, locale=locale, domain=domain)
finally:
infile.close()
# Get strings to translate.
# TODO(jeremydw): Use actual string, not the msgid. Currently we assume
# the msgid is the source string.
messages_to_translate = [message for message in babel_catalog
if not message.string]
strings_to_translate = [message.id for message in messages_to_translate]
if not strings_to_translate:
logging.info('No untranslated strings for {}, skipping.'.format(locale))
return
# Convert Python-format named placeholders to numerical placeholders
# compatible with Google Translate. Ex: %(name)s => (O).
placeholders = [] # Lists (#) placeholders to %(name)s placeholders.
for n, string in enumerate(strings_to_translate):
match = re.search('(%\([^\)]*\)\w)', string)
if not match:
placeholders.append(None)
continue
for i, group in enumerate(match.groups()):
num_placeholder = '({})'.format(i)
nums_to_names = {}
nums_to_names[num_placeholder] = group
replaced_string = string.replace(group, num_placeholder)
placeholders.append(nums_to_names)
strings_to_translate[n] = replaced_string
machine_translator = goslate.Goslate()
results = machine_translator.translate(strings_to_translate, locale)
for i, string in enumerate(results):
message = messages_to_translate[i]
# Replace numerical placeholders with named placeholders.
if placeholders[i]:
for num_placeholder, name_placeholder in placeholders[i].iteritems():
string = string.replace(num_placeholder, name_placeholder)
message.string = string
if isinstance(string, unicode):
string = string.encode('utf-8')
source = message.id
source = (source.encode('utf-8')
if isinstance(source, unicode) else source)
outfile = self.pod.open_file(self.pod_path, mode='w')
try:
pofile.write_po(outfile, babel_catalog, width=80)
finally:
outfile.close()
text = 'Machine translated {} strings: {}'
logging.info(text.format(len(strings_to_translate), self.pod_path))
def test_with_context(self):
buf = BytesIO(b'''# Some string in the menu
#: main.py:1
msgctxt "Menu"
msgid "foo"
msgstr "Voh"
# Another string in the menu
#: main.py:2
msgctxt "Menu"
msgid "bar"
msgstr "Bahr"
''')
catalog = pofile.read_po(buf, ignore_obsolete=True)
self.assertEqual(2, len(catalog))
message = catalog.get('foo', context='Menu')
self.assertEqual('Menu', message.context)
message = catalog.get('bar', context='Menu')
self.assertEqual('Menu', message.context)
# And verify it pass through write_po
out_buf = BytesIO()
pofile.write_po(out_buf, catalog, omit_header=True)
assert out_buf.getvalue().strip() == buf.getvalue().strip(), \
out_buf.getvalue()
def test_preserve_domain(self):
buf = StringIO(
r'''msgid "foo"
msgstr "Voh"'''
)
catalog = pofile.read_po(buf, domain="mydomain")
self.assertEqual("mydomain", catalog.domain)
def compile_catalogs(self, languages=None):
"""
Compile all knowed catalogs
"""
languages = languages or self.settings.LANGUAGES
for locale in languages:
catalog_path = self.get_catalog_path(locale)
self.logger.info('Compiling catalog (PO) for language {0} at {1}'.format(locale, catalog_path))
infile = open(catalog_path, 'r')
try:
catalog = read_po(infile, locale)
finally:
infile.close()
# Check errors in catalog
errs = False
for message, errors in catalog.check():
for error in errors:
errs = True
self.logger.warning('Error at line {0}: {1}'.format(message.lineno, error))
# Don't overwrite the previous MO file if there have been error
if errs:
self.logger.error('There has been errors within the catalog, compilation has been aborted')
break
outfile = open(self.get_compiled_catalog_path(locale), 'wb')
try:
write_mo(outfile, catalog, use_fuzzy=False)
finally:
outfile.close()
def test_with_context(self):
buf = StringIO(
r"""# Some string in the menu
#: main.py:1
msgctxt "Menu"
msgid "foo"
msgstr "Voh"
# Another string in the menu
#: main.py:2
msgctxt "Menu"
msgid "bar"
msgstr "Bahr"
"""
)
catalog = pofile.read_po(buf, ignore_obsolete=True)
self.assertEqual(2, len(catalog))
message = catalog.get("foo", context="Menu")
self.assertEqual("Menu", message.context)
message = catalog.get("bar", context="Menu")
self.assertEqual("Menu", message.context)
# And verify it pass through write_po
out_buf = BytesIO()
pofile.write_po(out_buf, catalog, omit_header=True)
assert out_buf.getvalue().strip() == buf.getvalue().strip().encode("latin-1"), out_buf.getvalue()
def test_preserve_locale(self):
buf = StringIO(
r'''msgid "foo"
msgstr "Voh"'''
)
catalog = pofile.read_po(buf, locale="en_US")
self.assertEqual(Locale("en", "US"), catalog.locale)
def load_i18n(project_root, tell_sentry):
# Load the locales
localeDir = os.path.join(project_root, 'i18n', 'core')
locales = LOCALES
for file in os.listdir(localeDir):
try:
parts = file.split(".")
if not (len(parts) == 2 and parts[1] == "po"):
continue
lang = parts[0]
with open(os.path.join(localeDir, file)) as f:
l = locales[lang.lower()] = Locale(lang)
c = l.catalog = read_po(f)
c.plural_func = get_function_from_rule(c.plural_expr)
try:
l.countries = make_sorted_dict(COUNTRIES, l.territories)
except KeyError:
l.countries = COUNTRIES
try:
l.languages_2 = make_sorted_dict(LANGUAGES_2, l.languages)
except KeyError:
l.languages_2 = LANGUAGES_2
except Exception as e:
tell_sentry(e, {})
# Add aliases
for k, v in list(locales.items()):
locales.setdefault(ALIASES.get(k, k), v)
locales.setdefault(ALIASES_R.get(k, k), v)
for k, v in list(locales.items()):
locales.setdefault(k.split('_', 1)[0], v)
# Patch the locales to look less formal
locales['fr'].currency_formats[None] = parse_pattern('#,##0.00\u202f\xa4')
locales['fr'].currency_symbols['USD'] = '$'
def load(self, pod_path=None):
# Use the "pod_path" argument to load another catalog (such as a template
# catalog) into this one.
if pod_path is None:
pod_path = self.pod_path
if not self.pod.file_exists(pod_path):
self.pod.write_file(pod_path, '')
po_file = self.pod.open_file(pod_path)
try:
babel_catalog = pofile.read_po(po_file, self.locale)
finally:
po_file.close()
attr_names = [
'_messages',
'_num_plurals',
'_plural_expr',
'charset',
'copyright_holder',
'creation_date',
'domain',
'fuzzy',
'language_team',
'last_translator',
'locale',
'msgid_bugs_address',
'obsolete',
'project',
'revision_date',
'version',
]
for name in attr_names:
setattr(self, name, getattr(babel_catalog, name))
def run(self):
if tx_executable is None:
sys.exit('Transifex client executable (tx) not found.')
from babel.messages import pofile
countries = dict()
if not self.skip_pull:
txpull_cmd = [
tx_executable,
'pull',
'--force',
'--resource=musicbrainz.countries',
'--source',
'--language=none',
]
self.spawn(txpull_cmd)
potfile = os.path.join('po', 'countries', 'countries.pot')
isocode_comment = u'iso.code:'
with open(potfile, 'rb') as f:
log.info('Parsing %s' % potfile)
po = pofile.read_po(f)
for message in po:
if not message.id or not isinstance(message.id, unicode):
continue
for comment in message.auto_comments:
if comment.startswith(isocode_comment):
code = comment.replace(isocode_comment, u'')
countries[code] = message.id
if countries:
self.countries_py_file(countries)
else:
sys.exit('Failed to extract any country code/name !')
def main():
# Check arguments
parser = argparse.ArgumentParser()
parser.add_argument('pot_filename', type=argparse.FileType('r'))
parser.add_argument('po_filename', type=argparse.FileType('w'))
parser.add_argument('locale')
args = parser.parse_args()
# read POT file
pot_cat = pofile.read_po(args.pot_filename, ignore_obsolete=True)
# Create the new Catalog
new_cat = catalog.Catalog(locale=args.locale,
last_translator="pseudo.py",
charset="utf-8")
num_plurals = new_cat.num_plurals
# Process messages from template
for msg in pot_cat:
if msg.pluralizable:
msg.string = [translate(u"{}:{}".format(i, msg.id[0]))
for i in range(num_plurals)]
else:
msg.string = translate(msg.id)
new_cat[msg.id] = msg
# Write "translated" PO file
pofile.write_po(args.po_filename, new_cat, ignore_obsolete=True)
def run(self):
log.info('translating catalog %r based on %r', self.output_file,
self.input_file)
infile = open(self.input_file, 'r')
try:
# Although reading from the catalog template, read_po must be fed
# the locale in order to correcly calculate plurals
catalog = read_po(infile, locale=self.locale)
finally:
infile.close()
catalog.locale = self._locale
catalog.fuzzy = False
for message in catalog:
if message.id:
# Recopie de l'id du message dans la traduction.
message.string = message.id
catalog[message.id] = message
outfile = open(self.output_file, 'w')
try:
write_po(outfile, catalog)
finally:
outfile.close()
def merge(source, target_filename):
"""Merges the messages from the source Catalog into a .po file at
target_filename. Creates the target file if it doesn't exist."""
if os.path.exists(target_filename):
target = pofile.read_po(open(target_filename))
for message in source:
if message.id and message.string and not message.fuzzy:
log_change(message.id in target and target[message.id], message)
# This doesn't actually replace the message! It just updates
# the fields other than the string. See Catalog.__setitem__.
target[message.id] = message
# We have to mutate the message to update the string and flags.
target[message.id].string = message.string
target[message.id].flags = message.flags
else:
for message in source:
log_change(None, message)
target = source
target_file = create_file(target_filename)
pofile.write_po(target_file, target,
no_location=True, sort_output=True, ignore_obsolete=True)
target_file.close()
def compile_domain(domain):
for locale in gettext_finder.GETTEXT_LANGUAGES:
popath = os.path.join('locale', locale, "LC_MESSAGES", domain + ".po")
mopath = os.path.join('locale', locale, "LC_MESSAGES", domain + ".mo")
with open(mopath, 'w') as mo_f, open(popath) as po_f:
write_mo(mo_f, read_po(po_f))
def main():
"""
Usage: i19json SOURCE LOCALE CACHE OUTPUT
SOURCE po file
LOCALE locale identifier
CACHE cache file created by i19extract.py
OUTPUT JSON file
"""
assert len(sys.argv) == 5, main.__doc__
po_file, locale, cache_file, jo_file = sys.argv[1:5]
getLogger().name = po_file
getLogger().level = 0
with file(po_file) as pof:
catalog = read_po(pof, locale)
messages, total, translated = catalog2dict(list(catalog)[1:], cache_file)
info("%s: %d of %d (%d unique) translated (%d%%)",
jo_file, translated, total,
len(messages), 100.0 * translated / total,)
messages['__pluralization_count__'], messages['__pluralization_expr__'] = \
extract_plural_func(catalog)
with file(jo_file, 'w') as json_file:
json.dump({locale: messages}, json_file)
def update_catalogs(self, languages=None):
"""
Update PO catalogs from POT
Keyword Arguments:
languages (list): List of languages to process. Default is
``None`` so languages are taken from ``LANGUAGES`` settings.
Returns:
list: List of language identifiers for updated catalogs.
"""
languages = self.parse_languages(languages or self.settings.LANGUAGES)
updated = []
for locale in languages:
catalog_path = self.get_po_filepath(locale)
msg = "Updating catalog (PO) for language '{0}' to {1}"
self.logger.info(msg.format(locale, catalog_path))
# Open PO file
with io.open(catalog_path, 'U') as fp:
catalog = read_po(fp, locale=locale)
# Update it from the template
catalog.update(self.pot)
self.safe_write_po(catalog, catalog_path)
updated.append(locale)
return updated
def _get_or_create_catalog(self, template_path):
exists = True
if not self.pod.file_exists(template_path):
self.pod.create_file(template_path, None)
exists = False
catalog = pofile.read_po(self.pod.open_file(template_path))
return catalog, exists
def write_mo_file(self):
"""
write MO files from existing PO files
"""
print('updating MO file...')
po_path = os.path.join(self.translations_path, self.language,
'LC_MESSAGES', self.translations_key + '.po')
po_file = open(po_path, 'r+')
catalog = read_po(po_file)
mo_path = os.path.join(self.translations_path, self.language,
'LC_MESSAGES', self.translations_key + '.mo')
mo_file = open(mo_path, 'w+')
write_mo(mo_file, catalog)
return
def run(self):
self.log.info(
'creating catalog %s based on %s', self.output_file, self.input_file
)
with open(self.input_file, 'rb') as infile:
# Although reading from the catalog template, read_po must be fed
# the locale in order to correctly calculate plurals
catalog = read_po(infile, locale=self.locale)
catalog.locale = self._locale
catalog.revision_date = datetime.now(LOCALTZ)
catalog.fuzzy = False
with open(self.output_file, 'wb') as outfile:
write_po(outfile, catalog, width=self.width)
def test_translation_coverage():
p = Path('./fava/translations/')
for po_file in list(p.glob('**/*.po')):
with po_file.open() as file:
catalog = read_po(file)
translated = 0
for message in list(catalog)[1:]:
if message.string:
translated += 1
assert translated / len(catalog) == 1, \
"Only {}% ({} of {}) messages are translated in {}".format(
translated * 100 // len(catalog), translated, len(catalog),
po_file
)
def clone_template(self):
"""
Open the template catalog again to clone it and to be able to modify it without
change on the "_catalog_template"
NOTE: does it invalidate get_catalog_template method and the _catalog_template
cache usage if after all it is not really usable
NOTE: seems in fact that processes does not really need to access to a verbatim
catalog template, so finally we could do cloning in catalog_template, then be
able modify it in extract without any loss for further process
"""
self.logger.debug('Opening template catalog (POT)')
fp = open(self.get_template_path(), "r")
catalog = read_po(fp)
fp.close()
return catalog
def test_with_previous(self):
buf = StringIO(r'''
#: main.py:1
#| msgctxt "f"
#| msgid "fo"
msgid "foo"
msgstr "Voh"
''')
catalog = pofile.read_po(buf)
self.assertEqual(1, len(catalog))
message = catalog["foo"]
self.assertEqual("foo", message.id)
self.assertEqual("Voh", message.string)
self.assertEqual([("main.py", 1)], message.locations)
self.assertEqual("f", message.previous_context)
self.assertEqual("fo", message.previous_id)
def get_modulemd_translations(translation_files,
debug=False):
catalogs = dict()
for f in translation_files:
with open(f, 'r') as infile:
catalog = pofile.read_po(infile)
catalogs[catalog.locale_identifier] = catalog
translations = get_modulemd_translations_from_catalog_dict(catalogs)
if debug:
for translation in translations:
print(translation.dumps())
return translations
def handle(self, *args, **options):
for module in options['module']:
for domain in options['domain']:
potfile = POTFILE.format(module=module, domain=domain)
for language in options['language']:
# Get the locale code for the language code given and
# work around broken django conversion function
locales = {'ko': 'ko_KR', 'pl': 'pl_PL', 'tr': 'tr_TR'}
locale = locales.get(language,
translation.to_locale(language))
pofile = POFILE.format(module=module, locale=locale,
domain=domain)
# If this isn't a pseudo translation, execute pybabel
if not options['pseudo']:
if not os.path.exists(pofile):
with open(pofile, 'wb') as fobj:
fobj.write(b'')
cmd = ('pybabel update -l {locale} -i {potfile} '
'-o {pofile}').format(locale=locale,
potfile=potfile,
pofile=pofile)
call(cmd, shell=True)
continue
# Pseudo translation logic
with open(potfile, 'r') as f:
pot_cat = babel_pofile.read_po(f, ignore_obsolete=True)
new_cat = catalog.Catalog(locale=locale,
last_translator="pseudo.py",
charset="utf-8")
num_plurals = new_cat.num_plurals
for msg in pot_cat:
if msg.pluralizable:
msg.string = [
translate(u"{}:{}".format(i, msg.id[0]))
for i in range(num_plurals)]
else:
msg.string = translate(msg.id)
new_cat[msg.id] = msg
with open(pofile, 'w') as f:
babel_pofile.write_po(f, new_cat, ignore_obsolete=True)
def do_compile_translations(target=("t", ""),
i18n_dir=("i", ""),
all=("a", False)):
"""
Compiling all the templates in specified application.
"""
if not target and not all:
print_status('Please specify target.')
sys.exit(1)
elif target == 'kay':
print_status('Compiling builtin languages')
root = path.join(kay.KAY_DIR, 'i18n')
elif all:
targets = get_user_apps()
for target in targets:
do_compile_translations(target=target, i18n_dir=None, all=False)
do_compile_translations(target=kay.PROJECT_DIR,
i18n_dir=None,
all=False)
sys.exit(0)
else:
if i18n_dir:
root = i18n_dir
else:
root = path.join(target, 'i18n')
if not path.isdir(root):
print('i18n folder missing')
sys.exit(1)
print_status('Compiling %s' % root)
for domain in domains:
for lang in listdir(root):
folder = path.join(root, lang)
translations = path.join(folder, 'LC_MESSAGES', domain + '.po')
if path.isfile(translations):
mo_file = open(translations.replace('.po', '.mo'), 'wb')
print_status('Compiling %s ' % translations)
f = file(translations)
try:
catalog = read_po(f, locale=lang)
finally:
f.close()
# Write standard catalog
write_mo(mo_file, catalog)
mo_file.close()
print_status('All done.')
def _compile(self, app):
for dirpath, dirnames, filenames in filtered_walk(
os.path.join(app.path, "locale")):
for filename in filenames:
filename = os.path.join(dirpath, filename)
print(filename)
if filename.endswith(".po"):
with open(filename, "r") as infp:
catalog = pofile.read_po(infp)
if not len(catalog):
continue
bio = io.BytesIO()
mofile.write_mo(bio, catalog)
mo_file = filename.replace(".po", ".mo")
with open(mo_file, "wb") as outfp:
outfp.write(bio.getvalue())
self.log.info("%s compiled to %s", filename, mo_file)
def test_obsolete_message(self):
buf = StringIO(r'''# This is an obsolete message
#~ msgid "foo"
#~ msgstr "Voh"
# This message is not obsolete
#: main.py:1
msgid "bar"
msgstr "Bahr"
''')
catalog = pofile.read_po(buf)
self.assertEqual(1, len(catalog))
self.assertEqual(1, len(catalog.obsolete))
message = catalog.obsolete[u'foo']
self.assertEqual(u'foo', message.id)
self.assertEqual(u'Voh', message.string)
self.assertEqual(['This is an obsolete message'], message.user_comments)
def open_po(self):
"""
Testing PO file opening with --source CLI argument
"""
if self.source_filepath:
fp = open(self.source_filepath, "r")
try:
catalog = read_po(fp)
except:
raise CommandError("Invalid PO file")
else:
print catalog.project
print json.dumps(catalog.mime_headers, indent=4)
finally:
fp.close()
else:
raise CommandError("Command action need the --source argument")
def pot(self):
"""
Return the catalog template
Get it from memory if allready opened, if allready exists then open
it, else extract it and create it.
Returns:
babel.messages.catalog.Catalog: Catalog template object.
"""
if self._pot is not None:
return self._pot
if self.check_template_path():
with io.open(self.get_template_path(), "rb") as fp:
self._pot = read_po(fp)
return self._pot
return self.build_pot()
def po_to_json(po_file, locale=None, domain=None):
"""Convert *.po file to a json-like data structure."""
with open(po_file, 'rb') as f:
po_data = read_po(f, locale=locale, domain=domain)
messages = dict((message.id[0], message.string) if message.pluralizable else (message.id, [message.string])
for message in po_data)
messages[''] = {
'domain': po_data.domain,
'lang': str(po_data.locale),
'plural_forms': po_data.plural_forms
}
return {
(po_data.domain or ''): messages
}
def compile_catalogs(self, languages=None):
"""
Compile PO catalogs to MO files
Note:
Errors have no test coverage since ``read_po()`` pass them through
warnings print to stdout and this is not blocking or detectable.
And so the code continue to the compile part.
Keyword Arguments:
languages (list): List of languages to process. Default is
``None`` so languages are taken from ``LANGUAGES`` settings.
Returns:
list: List of language identifiers for compiled catalogs.
"""
languages = self.parse_languages(languages or self.settings.LANGUAGES)
compiled = []
for locale in languages:
msg = "Compiling catalog (MO) for language '{0}' to {1}"
self.logger.info(msg.format(locale, self.get_mo_filepath(locale)))
with io.open(self.get_po_filepath(locale), "rb") as fp:
#
catalog = read_po(fp, locale)
# Check errors in catalog
errs = False
for message, errors in catalog.check():
for error in errors:
errs = True
self.logger.warning("Error at line {0}: {1}".format(
message.lineno, error))
# Don't overwrite previous MO file if there have been error
if errs:
self.logger.critical(("There has been errors within the "
"catalog, compilation has been aborted"))
break
with io.open(self.get_mo_filepath(locale), "wb") as fp:
write_mo(fp, catalog, use_fuzzy=False)
compiled.append(locale)
return compiled
def compile_languages(self, remove_po_files=False):
for filename in glob.glob('translations/*/LC_MESSAGES/messages.po'):
language = filename[
len('translations/'):-len('/LC_MESSAGES/messages.po')
]
Logger.info('Translator: Compiling language %s...', language)
with open(filename, 'rb') as po_file:
catalog = pofile.read_po(po_file, locale=language)
mo_filename = filename.replace('.po', '.mo')
with open(mo_filename, 'wb') as mo_file:
mofile.write_mo(mo_file, catalog)
if remove_po_files:
os.remove(filename)
def validate_pot(self, attrs, source):
"""
Validation for the given POT content
"""
value = attrs[source]
if value:
try:
template_file = StringIO()
template_file.write(value.encode('UTF8'))
template_file.seek(0)
# Seems the validation from read_po is too much minimalistic
# This does not really valid if the content is a real POT content
self.uploaded_pot_file = read_po(template_file,
ignore_obsolete=True)
except:
raise serializers.ValidationError(
"Your file does not seem to be a valid POT file")
return attrs
def update(self, template_path=None, use_fuzzy_matching=False,
include_obsolete=False, include_header=False):
"""Updates catalog with messages from a template."""
if template_path is None:
template_path = os.path.join('translations', 'messages.pot')
if not self.exists:
self.init(template_path)
return
template_file = self.pod.open_file(template_path)
template = pofile.read_po(template_file)
super(Catalog, self).update(
template, no_fuzzy_matching=(not use_fuzzy_matching))
# Don't use gettext's obsolete functionality as it polutes files: merge
# into main translations if anything
if include_obsolete:
self.merge_obsolete()
self.obsolete = odict()
self.save(include_header=include_header)
def message_catalog(config: SDConfig, locale: str) -> Catalog:
"""
Returns the gettext message catalog for the given locale.
With the catalog, tests can check whether a gettext call returned
an actual translation or merely the result of falling back to the
default locale.
>>> german = message_catalog(config, 'de_DE')
>>> m = german.get("a string that has been added to the catalog but not translated")
>>> m.string
''
>>> german.get("Password").string
'Passwort'
"""
return read_po(
open(str(config.TRANSLATION_DIRS / locale /
"LC_MESSAGES/messages.po")))
def main():
try:
import jieba
except ImportError:
return
pofile.normalize = _normalize
for root, dirs, files in os.walk('.'):
if 'zh' not in root:
continue
for f in files:
if not f.endswith('.po'):
continue
path = os.path.join(root, f)
with open(path, 'rb') as inpf:
catalog = pofile.read_po(inpf)
with open(path, 'wb') as outf:
pofile.write_po(outf, catalog)
def po_catalog_export(self):
"""
Export a catalog from a project into a PO file
"""
toast_path = "/home/django/Emencia/po_headquarter/dummy_po/"
project = Project.objects.get(slug='dummy')
catalog = Catalog.objects.get(project=project, locale='fr')
#mime_dict = json.loads(catalog.mime_headers)
forged_catalog = BabelCatalog(
locale=catalog.locale,
header_comment=catalog.header_comment,
project=project.name,
version="0.2.0"
)
print "before add:", len(forged_catalog)
for entry in catalog.translationmsg_set.all().order_by('id'):
locations = [tuple(item) for item in json.loads(entry.template.locations)]
forged_catalog.add(entry.template.message, string=entry.message, locations=locations, flags=entry.template.flags)
print "after add:", len(forged_catalog)
print "errors:", [item for item in forged_catalog.check()]
print
print "---------------- Original"
fpw = StringIO()
write_po(fpw, forged_catalog, sort_by_file=False, ignore_obsolete=True, include_previous=False)
print fpw.getvalue()
fpw.close()
print
print "---------------- Updated"
fp3 = open(os.path.join(toast_path, '0-3-0.pot'), 'r')
template_catalog_3 = read_po(fp3)
forged_catalog.update(template_catalog_3)
fpw = StringIO()
write_po(fpw, forged_catalog, sort_by_file=False, ignore_obsolete=True, include_previous=False)
print fpw.getvalue()
fpw.close()
def add_translation_notes():
"""
Implements the localization policy.
"Minor, non-normative, documentation updates will be translated promptly, but may not always be translated before
the updates are released. The documentation will clearly display when the English documentation is 'ahead' of
translations for a particular version."
https://standard.open-contracting.org/1.1/en/governance/#translation-and-localization-policy
"""
excluded = ('.doctrees', '_downloads', '_images', '_sources', '_static', 'codelists', 'genindex', 'search')
for language in ('es', 'fr'):
build_dir = basedir / 'build' / language
language_dir = localedir / language / 'LC_MESSAGES'
for root, dirs, files in os.walk(build_dir):
# Skip Sphinx directories.
for directory in excluded:
if directory in dirs:
dirs.remove(directory)
if root == str(build_dir):
continue
for name in files:
# See `sphinx.transforms.i18n.Locale.apply()`.
# https://github.com/sphinx-doc/sphinx/blob/v2.2.1/sphinx/transforms/i18n.py
source = os.path.join(root, os.path.dirname(name))
domain = relative_path(build_dir, source)
path = language_dir / domain / 'index.po'
if not path.is_file():
path = language_dir / f'{domain}.po'
if not path.is_file():
add_translation_note(os.path.join(root, name), language, domain)
continue
# Check the PO files, because Babel sets the msgstr to the msgid if the msgstr is missing.
with open(path) as f:
for message in read_po(f):
if not message.string:
add_translation_note(os.path.join(root, name), language, domain)
break
def load_i18n(project_root, tell_sentry):
# Load the locales
key = lambda t: strip_accents(t[1])
localeDir = os.path.join(project_root, 'i18n', 'core')
locales = i18n.LOCALES
for file in os.listdir(localeDir):
try:
parts = file.split(".")
if not (len(parts) == 2 and parts[1] == "po"):
continue
lang = parts[0]
with open(os.path.join(localeDir, file)) as f:
l = locales[lang.lower()] = Locale(lang)
c = l.catalog = read_po(f)
c.plural_func = get_function_from_rule(c.plural_expr)
try:
l.countries_map = {
k: l.territories[k]
for k in COUNTRIES_MAP
}
l.countries = sorted(l.countries_map.items(), key=key)
except KeyError:
l.countries_map = COUNTRIES_MAP
l.countries = COUNTRIES
except Exception as e:
tell_sentry(e)
# Add the default English locale
locale_en = i18n.LOCALE_EN = locales['en'] = Locale('en')
locale_en.catalog = Catalog('en')
locale_en.catalog.plural_func = lambda n: n != 1
locale_en.countries = COUNTRIES
locale_en.countries_map = COUNTRIES_MAP
# Add aliases
for k, v in list(locales.items()):
locales.setdefault(ALIASES.get(k, k), v)
locales.setdefault(ALIASES_R.get(k, k), v)
for k, v in list(locales.items()):
locales.setdefault(k.split('_', 1)[0], v)
# Patch the locales to look less formal
locales['fr'].currency_formats[None] = parse_pattern('#,##0.00\u202f\xa4')
locales['fr'].currency_symbols['USD'] = '$'
def gen_statistics(input_dir, output_file):
"""
Generate statistics on languages for how translated they are.
Keyword arguments:
- input_dir: The directory of languages we'll iterate through
- output_file: Path to the CSV file that will be written to
"""
output_file = file(output_file, 'w')
input_dir = os.path.abspath(input_dir)
lang_dirs = os.listdir(input_dir)
# Create CSV writer
writer = csv.DictWriter(output_file, CSV_HEADERS)
# iterate through all the languages
for lang in lang_dirs:
trans_file = os.path.join(input_dir, lang, 'cc_org.po')
if not os.path.exists(trans_file):
continue
# load .po file
pofile = read_po(file(trans_file, 'r'))
fuzzies = 0
translated = 0
# generate statistics
for message in pofile:
if message.string:
if message.fuzzy:
fuzzies += 1
else:
translated += 1
writer.writerow(
{'lang': lang,
'num_messages': len(pofile),
'num_trans': translated,
'num_fuzzy': fuzzies,
'percent_trans': int((float(translated) / len(pofile)) * 100)})
output_file.close()
def build_catalog(self):
"""
build catalog off of existing PO files
"""
print('building catalog off of existing translations in PO file...')
po_path = os.path.join(self.translations_path, self.language,
'LC_MESSAGES', self.translations_key + '.po')
try:
file = open(po_path, 'r+')
except IOError as e:
# po file doesn't exist, create a blank file
if not os.path.exists(os.path.dirname(po_path)):
os.makedirs(os.path.dirname(po_path))
file = open(po_path, 'w+')
catalog = read_po(file)
for message in catalog:
if message.id:
self.catalog.add(message.id, message.string)
return
def test_missing_plural_in_the_middle(self):
buf = StringIO('''\
msgid ""
msgstr ""
"Plural-Forms: nplurals=3; plural=(n < 2) ? n : 2\n"
msgid "foo"
msgid_plural "foos"
msgstr[0] "Voh [text]"
msgstr[2] "Vohs [text]"
''')
catalog = pofile.read_po(buf, locale='nb_NO')
self.assertEqual(1, len(catalog))
self.assertEqual(3, catalog.num_plurals)
message = catalog['foo']
self.assertEqual(3, len(message.string))
self.assertEqual("Voh [text]", message.string[0])
self.assertEqual("", message.string[1])
self.assertEqual("Vohs [text]", message.string[2])
def loadCatalogs(source_dir):
"""Load the translation catalogs and return a dictionary mapping
the locale code to the PoFile object."""
langs = {}
for root, dirnames, filenames in os.walk(source_dir):
for fn in filenames:
if fn[-3:] == ".po":
# figure out what locale this is based on pathname
locale = root.split(os.sep)[-1]
print "loading catalog for {}...".format(locale)
msg_catalog = read_po(
file(os.path.abspath(os.path.join(root, fn)), "r"))
langs[locale] = msg_catalog
return langs
def update(self,
use_fuzzy=False,
ignore_obsolete=True,
include_previous=True,
width=80):
if not self.exists:
self.init()
return
# Updates with new extracted messages from the template.
template_file = self.pod.open_file(
os.path.join('translations', 'messages.pot'))
template = pofile.read_po(template_file)
super(Catalog, self).update(template, use_fuzzy)
# Save the result.
self.save(ignore_obsolete=ignore_obsolete,
include_previous=include_previous,
width=width)
def update_catalogs(self, languages=None):
"""
Update all knowed catalogs from the catalog template
"""
languages = self.parse_languages(languages or self.settings.LANGUAGES)
for locale in languages:
catalog_path = self.get_catalog_path(locale)
self.logger.info(
'Updating catalog (PO) for language {0} at {1}'.format(
locale, catalog_path))
# Open the PO file
infile = open(catalog_path, 'U')
try:
catalog = read_po(infile, locale=locale)
finally:
infile.close()
# Update it from the template
catalog.update(self.catalog_template)
self.safe_write_po(catalog, catalog_path)
def build_translations(self, recompile_translations=False):
"""Create MO files for existing PO translation files."""
for ld in self.languages:
lang_folder = os.path.join(ld, "LC_MESSAGES")
outfile_folder = os.path.join(paths.CACHE_DIR, "l18n", lang_folder)
infile = os.path.join(prepare.fetch("l18n"), lang_folder,
"base.po")
outfile = os.path.join(outfile_folder, "base.mo")
# build only complete translations
if os.path.exists(infile) and (not os.path.exists(outfile)
or recompile_translations):
with open(infile, "r", encoding="UTF8") as po_file:
catalog = read_po(po_file)
os.makedirs(outfile_folder, exist_ok=True)
with open(outfile, "wb") as mo_file:
write_mo(mo_file, catalog)
logger.debug("writing l18n mo: %s", outfile)
def test_obsolete_message_ignored(self):
buf = StringIO(r'''# User comment
#. Developer Comment
#: utils.py:3
#, fuzzy
#| msgctxt "previous context"
#| msgid "foo"
#~ msgctxt "context"
#~ msgid "bar"
#~ msgstr "Bahr"
# This message is not obsolete
#: main.py:1
msgid "bar"
msgstr "Bahr"
''')
catalog = pofile.read_po(buf, ignore_obsolete=True)
self.assertEqual(1, len(catalog))
self.assertEqual(0, len(catalog.obsolete))
