def compile(self):
self.pod.catalogs.clear_gettext_cache()
localization = self.pod.podspec.localization
if localization is None:
return
compile_fuzzy = localization.get('compile_fuzzy')
mo_filename = self.mo_path
num_translated = 0
num_total = 0
for message in list(self)[1:]:
if message.string:
num_translated += 1
num_total += 1
try:
for message, errors in self.check():
for error in errors:
text = 'Error compiling ({}:{}): {}'
message = text.format(self.locale, message.lineno, error)
self.pod.logger.error(message)
except IOError:
self.pod.logger.info('Skipped catalog check for: {}'.format(self))
text = 'Compiled: {} ({}/{})'
self.pod.logger.info(text.format(self.locale, num_translated, num_total))
mo_file = self.pod.open_file(mo_filename, 'w')
try:
mofile.write_mo(mo_file, self, use_fuzzy=compile_fuzzy)
finally:
mo_file.close()
def get_po_catalog(self):
cat = StringIO()
with closing(zipopen(self.file_base + '.po')) as f:
po = read_po(f)
write_mo(cat, po)
cat.seek(0)
return cat
def test_translation_plugins(app, tmpdir):
session.lang = 'fr_FR'
plugin = MockPlugin(plugin_engine, app)
app.extensions['pluginengine'].plugins['dummy'] = plugin
plugin.root_path = tmpdir.strpath
french_core_str = DICTIONARIES['fr_FR']['This is not a string']
french_plugin_str = "This is not le french string"
trans_dir = os.path.join(plugin.root_path, 'translations', 'fr_FR', 'LC_MESSAGES')
os.makedirs(trans_dir)
# Create proper *.mo file for plugin translation
with open(os.path.join(trans_dir, 'messages.mo'), 'wb') as f:
catalog = Catalog(locale='fr_FR', domain='plugin')
catalog.add("This is not a string", "This is not le french string")
write_mo(f, catalog)
gettext_plugin = make_bound_gettext('dummy')
assert _(u'This is not a string') == french_core_str
assert gettext_context(u"This is not a string") == french_core_str
assert gettext_plugin(u"This is not a string") == french_plugin_str
with plugin.plugin_context():
assert _(u'This is not a string') == french_core_str
assert gettext_context(u"This is not a string") == french_plugin_str
assert gettext_plugin(u"This is not a string") == french_plugin_str
def generate_mo (self, file_name, use_fuzzy = False) :
self._make_dir (file_name)
file = open (file_name, 'wb')
try:
write_mo (file, self.catalog, use_fuzzy = use_fuzzy)
finally:
file.close ()
def test_headers(self):
for x in range(1, 7):
text = "{0} This is a h{1}".format('#' * x, x)
expected = '<h{0} id="esto-es-un-h{0}">Esto es un h{0}</h{0}>'.format(x)
with TempDir() as d:
c = catalog.Catalog(locale='es_ES')
c.add('This is a h{0}'.format(x), 'Esto es un h{0}'.format(x))
os.mkdir(os.path.join(d.dir, 'es_ES'))
lc_messages = os.path.join(d.dir, 'es_ES', 'LC_MESSAGES')
os.mkdir(lc_messages)
mo_file = os.path.join(lc_messages, 'messages.mo')
with open(mo_file, 'w') as f:
mofile.write_mo(f, c)
result = self.markdown(
text,
extensions=['markdown.extensions.toc'],
extension_configs={
'markdown_i18n': {
'i18n_dir': d.dir,
'i18n_lang': 'es_ES'
}
}
)
self.assertEqual(expected, result)
def test_sorting(self):
# Ensure the header is sorted to the first entry so that its charset
# can be applied to all subsequent messages by GNUTranslations
# (ensuring all messages are safely converted to unicode)
catalog = Catalog(locale="en_US")
catalog.add(
u"",
"""\
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n""",
)
catalog.add(u"foo", "Voh")
catalog.add((u"There is", u"There are"), (u"Es gibt", u"Es gibt"))
catalog.add(u"Fizz", "")
catalog.add(("Fuzz", "Fuzzes"), ("", ""))
buf = StringIO()
mofile.write_mo(buf, catalog)
buf.seek(0)
translations = gettext.GNUTranslations(fp=buf)
self.assertEqual(u"Voh", translations.ugettext("foo"))
assert isinstance(translations.ugettext("foo"), unicode)
self.assertEqual(u"Es gibt", translations.ungettext("There is", "There are", 1))
assert isinstance(translations.ungettext("There is", "There are", 1), unicode)
self.assertEqual(u"Fizz", translations.ugettext("Fizz"))
assert isinstance(translations.ugettext("Fizz"), unicode)
self.assertEqual(u"Fuzz", translations.ugettext("Fuzz"))
assert isinstance(translations.ugettext("Fuzz"), unicode)
self.assertEqual(u"Fuzzes", translations.ugettext("Fuzzes"))
assert isinstance(translations.ugettext("Fuzzes"), unicode)
def setUp(self):
# Use a locale which won't fail to run the tests
os.environ["LANG"] = "en_US.UTF-8"
messages1 = [
("foo", {"string": "Voh"}),
("foo", {"string": "VohCTX", "context": "foo"}),
(("foo1", "foos1"), {"string": ("Voh1", "Vohs1")}),
(("foo1", "foos1"), {"string": ("VohCTX1", "VohsCTX1"), "context": "foo"}),
]
messages2 = [
("foo", {"string": "VohD"}),
("foo", {"string": "VohCTXD", "context": "foo"}),
(("foo1", "foos1"), {"string": ("VohD1", "VohsD1")}),
(("foo1", "foos1"), {"string": ("VohCTXD1", "VohsCTXD1"), "context": "foo"}),
]
catalog1 = Catalog(locale="en_GB", domain="messages")
catalog2 = Catalog(locale="en_GB", domain="messages1")
for ids, kwargs in messages1:
catalog1.add(ids, **kwargs)
for ids, kwargs in messages2:
catalog2.add(ids, **kwargs)
catalog1_fp = BytesIO()
catalog2_fp = BytesIO()
write_mo(catalog1_fp, catalog1)
catalog1_fp.seek(0)
write_mo(catalog2_fp, catalog2)
catalog2_fp.seek(0)
translations1 = support.Translations(catalog1_fp)
translations2 = support.Translations(catalog2_fp, domain="messages1")
self.translations = translations1.add(translations2, merge=False)
def run(self, root):
i18n_dir = self.extension.getConfig('i18n_dir')
pot_path = os.path.join(i18n_dir, 'messages.pot')
if os.path.exists(pot_path):
with open(pot_path, 'r') as f:
catalog = pofile.read_po(f)
else:
catalog = Catalog()
lang = self.extension.getConfig('i18n_lang')
mo_path = os.path.join(i18n_dir, lang, 'LC_MESSAGES', 'messages.mo')
po_path = os.path.join(i18n_dir, lang, 'LC_MESSAGES', 'messages.po')
if os.path.exists(po_path):
with open(po_path, 'r') as f:
lang_catalog = pofile.read_po(f)
with open(mo_path, 'w') as mo:
mofile.write_mo(mo, lang_catalog)
translations = Translations.load(i18n_dir, locales=[lang])
self.translate(catalog, translations, root)
with open(pot_path, 'w') as pot_file:
pofile.write_po(pot_file, catalog)
def recompile_mo(self, use_fuzzy=False):
locale = str(self.locale)
po_filename = os.path.join(self.path, 'LC_MESSAGES', 'messages.po')
mo_filename = os.path.join(self.path, 'LC_MESSAGES', 'messages.mo')
po_file = self.pod.open_file(po_filename)
try:
catalog = pofile.read_po(po_file, locale)
finally:
po_file.close()
num_translated = 0
num_total = 0
for message in list(catalog)[1:]:
if message.string:
num_translated += 1
num_total += 1
if catalog.fuzzy and not use_fuzzy:
logging.info('Catalog {} is marked as fuzzy, skipping.'.format(po_filename))
try:
for message, errors in catalog.check():
for error in errors:
logging.error('Error: {}:{}: {}'.format(po_filename, message.lineno, error))
except IOError:
logging.info('Skipped catalog check.')
text = 'Compiling {}/{} translated strings to {}'
logging.info(text.format(num_translated, num_total, mo_filename))
mo_file = self.pod.open_file(mo_filename, 'w')
try:
mofile.write_mo(mo_file, catalog, use_fuzzy=use_fuzzy)
finally:
mo_file.close()
def compile_domain(domain):
for locale in gettext_finder.GETTEXT_LANGUAGES:
popath = os.path.join('locale', locale, "LC_MESSAGES", domain + ".po")
mopath = os.path.join('locale', locale, "LC_MESSAGES", domain + ".mo")
with open(mopath, 'w') as mo_f, open(popath) as po_f:
write_mo(mo_f, read_po(po_f))
def compile_catalogs(self, languages=None):
"""
Compile all knowed catalogs
"""
languages = languages or self.settings.LANGUAGES
for locale in languages:
catalog_path = self.get_catalog_path(locale)
self.logger.info('Compiling catalog (PO) for language {0} at {1}'.format(locale, catalog_path))
infile = open(catalog_path, 'r')
try:
catalog = read_po(infile, locale)
finally:
infile.close()
# Check errors in catalog
errs = False
for message, errors in catalog.check():
for error in errors:
errs = True
self.logger.warning('Error at line {0}: {1}'.format(message.lineno, error))
# Don't overwrite the previous MO file if there have been error
if errs:
self.logger.error('There has been errors within the catalog, compilation has been aborted')
break
outfile = open(self.get_compiled_catalog_path(locale), 'wb')
try:
write_mo(outfile, catalog, use_fuzzy=False)
finally:
outfile.close()
def compile(self, use_fuzzy=False):
mo_dirpath = os.path.dirname(self.pod_path)
mo_filename = os.path.join(mo_dirpath, 'messages.mo')
num_translated = 0
num_total = 0
for message in list(self)[1:]:
if message.string:
num_translated += 1
num_total += 1
try:
for message, errors in self.check():
for error in errors:
logging.error('Error: {}:{}: {}'.format(self.locale, message.lineno, error))
except IOError:
logging.info('Skipped catalog check for: {}'.format(self))
text = 'Compiled: {} ({}/{})'
self.pod.logger.info(text.format(self.locale, num_translated, num_total))
mo_file = self.pod.open_file(mo_filename, 'w')
try:
mofile.write_mo(mo_file, self, use_fuzzy=use_fuzzy)
finally:
mo_file.close()
def test_sorting(self):
# Ensure the header is sorted to the first entry so that its charset
# can be applied to all subsequent messages by GNUTranslations
# (ensuring all messages are safely converted to unicode)
catalog = Catalog(locale='en_US')
catalog.add(u'', '''\
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n''')
catalog.add(u'foo', 'Voh')
catalog.add((u'There is', u'There are'), (u'Es gibt', u'Es gibt'))
catalog.add(u'Fizz', '')
catalog.add(('Fuzz', 'Fuzzes'), ('', ''))
buf = StringIO()
mofile.write_mo(buf, catalog)
buf.seek(0)
translations = gettext.GNUTranslations(fp=buf)
self.assertEqual(u'Voh', translations.ugettext('foo'))
assert isinstance(translations.ugettext('foo'), unicode)
self.assertEqual(u'Es gibt', translations.ungettext('There is', 'There are', 1))
assert isinstance(translations.ungettext('There is', 'There are', 1), unicode)
self.assertEqual(u'Fizz', translations.ugettext('Fizz'))
assert isinstance(translations.ugettext('Fizz'), unicode)
self.assertEqual(u'Fuzz', translations.ugettext('Fuzz'))
assert isinstance(translations.ugettext('Fuzz'), unicode)
self.assertEqual(u'Fuzzes', translations.ugettext('Fuzzes'))
assert isinstance(translations.ugettext('Fuzzes'), unicode)
def setUp(self):
# Use a locale which won't fail to run the tests
os.environ['LANG'] = 'en_US.UTF-8'
messages1 = [
('foo', {'string': 'Voh'}),
('foo', {'string': 'VohCTX', 'context': 'foo'}),
(('foo1', 'foos1'), {'string': ('Voh1', 'Vohs1')}),
(('foo1', 'foos1'), {'string': ('VohCTX1', 'VohsCTX1'), 'context': 'foo'}),
]
messages2 = [
('foo', {'string': 'VohD'}),
('foo', {'string': 'VohCTXD', 'context': 'foo'}),
(('foo1', 'foos1'), {'string': ('VohD1', 'VohsD1')}),
(('foo1', 'foos1'), {'string': ('VohCTXD1', 'VohsCTXD1'), 'context': 'foo'}),
]
catalog1 = Catalog(locale='en_GB', domain='messages')
catalog2 = Catalog(locale='en_GB', domain='messages1')
for ids, kwargs in messages1:
catalog1.add(ids, **kwargs)
for ids, kwargs in messages2:
catalog2.add(ids, **kwargs)
catalog1_fp = BytesIO()
catalog2_fp = BytesIO()
write_mo(catalog1_fp, catalog1)
catalog1_fp.seek(0)
write_mo(catalog2_fp, catalog2)
catalog2_fp.seek(0)
translations1 = support.Translations(catalog1_fp)
translations2 = support.Translations(catalog2_fp, domain='messages1')
self.translations = translations1.add(translations2, merge=False)
def _run_domain(self, domain):
po_files = []
mo_files = []
if not self.input_file:
if self.locale:
po_files.append((self.locale, os.path.join(self.directory, self.locale, "LC_MESSAGES", domain + ".po")))
mo_files.append(os.path.join(self.directory, self.locale, "LC_MESSAGES", domain + ".mo"))
else:
for locale in os.listdir(self.directory):
po_file = os.path.join(self.directory, locale, "LC_MESSAGES", domain + ".po")
if os.path.exists(po_file):
po_files.append((locale, po_file))
mo_files.append(os.path.join(self.directory, locale, "LC_MESSAGES", domain + ".mo"))
else:
po_files.append((self.locale, self.input_file))
if self.output_file:
mo_files.append(self.output_file)
else:
mo_files.append(os.path.join(self.directory, self.locale, "LC_MESSAGES", domain + ".mo"))
if not po_files:
raise DistutilsOptionError("no message catalogs found")
for idx, (locale, po_file) in enumerate(po_files):
mo_file = mo_files[idx]
infile = open(po_file, "rb")
try:
catalog = read_po(infile, locale)
finally:
infile.close()
if self.statistics:
translated = 0
for message in list(catalog)[1:]:
if message.string:
translated += 1
percentage = 0
if len(catalog):
percentage = translated * 100 // len(catalog)
self.log.info(
"%d of %d messages (%d%%) translated in %r", translated, len(catalog), percentage, po_file
)
if catalog.fuzzy and not self.use_fuzzy:
self.log.info("catalog %r is marked as fuzzy, skipping", po_file)
continue
for message, errors in catalog.check():
for error in errors:
self.log.error("error: %s:%d: %s", po_file, message.lineno, error)
self.log.info("compiling catalog %r to %r", po_file, mo_file)
outfile = open(mo_file, "wb")
try:
write_mo(outfile, catalog, use_fuzzy=self.use_fuzzy)
finally:
outfile.close()
def catalog_to_translations(catalog):
"""
Helper function which converts catalog object to translation
"""
buf = BytesIO()
write_mo(buf, catalog, use_fuzzy=True)
buf.seek(0)
return Translations(fp=buf)
def _compile_translation(po, mo):
try:
catalog = read_po(open(po))
f = open(mo, 'wb')
write_mo(f, catalog)
f.close()
print >> web.debug, 'compiled', po
except:
print >> web.debug, 'failed to compile', po
def _compile_translation(po, mo):
try:
catalog = read_po(open(po, 'rb'))
f = open(mo, 'wb')
write_mo(f, catalog)
f.close()
print('compiled', po, file=web.debug)
except Exception as e:
print('failed to compile', po, file=web.debug)
raise e
def inject_translations(self, locale, content):
po_file_to_merge = cStringIO.StringIO()
po_file_to_merge.write(content)
po_file_to_merge.seek(0)
catalog_to_merge = pofile.read_po(po_file_to_merge, locale)
mo_fp = cStringIO.StringIO()
mofile.write_mo(mo_fp, catalog_to_merge)
mo_fp.seek(0)
translation_obj = support.Translations(mo_fp, domain='messages')
self._gettext_translations[locale] = translation_obj
self.pod.logger.info('Injected translations -> {}'.format(locale))
def create_i18n_files(lang, quiet=True):
sql = """
SELECT * FROM translation
WHERE lang = ? and active=1;
"""
result = conn.execute(sql, lang)
catalog = Catalog(locale=lang)
for row in result:
if row.plural:
key = (row.id, row.plural)
values = [
row.trans1,
row.trans2,
row.trans3,
row.trans4,
]
value = []
for v in values:
if v:
value.append(v)
if len(value) == 1:
value = value[0]
else:
value = tuple(value)
else:
key = row.id
value = row.trans1
catalog.add(key, value)
path = os.path.join(DIR, 'translations', lang, 'LC_MESSAGES')
try:
os.makedirs(path)
except OSError:
pass
outfile = open(os.path.join(path, 'messages.po'), 'wb')
try:
if not quiet:
print 'writing PO for', lang
write_po(outfile, catalog)
finally:
outfile.close()
outfile = open(os.path.join(path, 'messages.mo'), 'wb')
try:
if not quiet:
print 'writing MO for', lang
write_mo(outfile, catalog)
finally:
outfile.close()
def inject_translations(self, locale, content):
po_file_to_merge = cStringIO.StringIO()
po_file_to_merge.write(content)
po_file_to_merge.seek(0)
catalog_to_merge = pofile.read_po(po_file_to_merge, locale)
mo_fp = cStringIO.StringIO()
mofile.write_mo(mo_fp, catalog_to_merge)
mo_fp.seek(0)
translation_obj = support.Translations(mo_fp, domain='messages')
self._gettext_translations[locale] = translation_obj
self.pod.logger.info('Injected translations -> {}'.format(locale))
def write_mo(filename, catalog):
"""write mo file from catalog object
:param unicode filename: path to mo file
:param catalog: catalog object
:return: None
"""
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
os.makedirs(dirname)
with io.open(filename, 'wb') as f:
mofile.write_mo(f, catalog)
def write_mo(filename, catalog):
"""write mo file from catalog object
:param unicode filename: path to mo file
:param catalog: catalog object
:return: None
"""
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
os.makedirs(dirname)
with io.open(filename, 'wb') as f:
mofile.write_mo(f, catalog)
def po_project_export(project, project_version, archive_fileobj, compile_mo=True):
"""
Export all catalogs from a project into PO files with the good directory
structure
NOTE: One improvement would be to not perform two querysets for each catalog (one for PO, one 'solid' for MO)
"""
archive_files = []
# Template catalog POT file
template_file = StringIO()
write_po(template_file, project_version.get_babel_template(), sort_by_file=False, ignore_obsolete=True, include_previous=False)
template_file.seek(0)
archive_files.append( (settings.POT_ARCHIVE_PATH.format(catalog_filename=project.domain), template_file) )
# Catalog PO files
for catalog in project_version.catalog_set.all():
po_file_path = settings.PO_ARCHIVE_PATH.format(locale=catalog.locale, catalog_filename=project.domain)
mo_file_path = settings.MO_ARCHIVE_PATH.format(locale=catalog.locale, catalog_filename=project.domain)
# Write the PO to a buffer string
po_file = StringIO()
write_po(po_file, catalog.get_babel_catalog(), sort_by_file=False, ignore_obsolete=True, include_previous=False)
po_file.seek(0)
# Append the PO file to the archive manifest
archive_files.append( (po_file_path, po_file) )
# Write the MO to a buffer string
mo_file = StringIO()
write_mo(mo_file, catalog.get_babel_catalog(solid=True), use_fuzzy=False)
mo_file.seek(0)
# Append the MO file to the archive manifest
archive_files.append( (mo_file_path, mo_file) )
# Open and fill tarball archive
archive = tarfile.open("{0}.tar.gz".format(project.slug), mode="w:gz", fileobj=archive_fileobj)
mtime = time.time()
# Build the tarball from the manifest
for name,content in archive_files:
info = tarfile.TarInfo(name)
info.size=len(content.getvalue())
info.mtime = mtime
archive.addfile(tarinfo=info, fileobj=content)
#
content.close()
archive.close()
return archive_fileobj
def setUpClass(cls):
"""Create temporary workspace and write MO file to it."""
cls.workspace_dir = tempfile.mkdtemp()
cls.test_locale_dir = os.path.join(cls.workspace_dir, 'locales')
mo_path = os.path.join(cls.test_locale_dir, TEST_LANG, 'LC_MESSAGES',
'messages.mo')
os.makedirs(os.path.dirname(mo_path))
# format the test catalog object into MO file format
with open(mo_path, 'wb') as mo_file:
mofile.write_mo(mo_file, TEST_CATALOG)
# reset so that these tests are unaffected by previous tests
reset_global_context()
def _compile_file(path, component, locale):
with io.open(str(path), 'r') as po:
catalog = read_po(po, locale=locale, domain=component)
logger.info("Compiling component [%s] locale [%s] (%d messages)...",
component, locale, len(catalog))
with io.open(str(path.with_suffix('.mo')), 'wb') as mo:
write_mo(mo, catalog)
with io.open(str(path.with_suffix('.jed')), 'w') as jed:
write_jed(jed, catalog)
def po_project_export(project, project_version, archive_fileobj, compile_mo=True):
"""
Export all catalogs from a project into PO files with the good directory
structure
NOTE: One improvement would be to not perform two querysets for each catalog (one for PO, one 'solid' for MO)
"""
archive_files = []
# Template catalog POT file
template_file = StringIO()
write_po(template_file, project_version.get_babel_template(), sort_by_file=False, ignore_obsolete=True, include_previous=False)
template_file.seek(0)
archive_files.append( (settings.POT_ARCHIVE_PATH.format(catalog_filename=project.domain), template_file) )
# Catalog PO files
for catalog in project_version.catalog_set.all():
po_file_path = settings.PO_ARCHIVE_PATH.format(locale=catalog.locale, catalog_filename=project.domain)
mo_file_path = settings.MO_ARCHIVE_PATH.format(locale=catalog.locale, catalog_filename=project.domain)
# Write the PO to a buffer string
po_file = StringIO()
write_po(po_file, catalog.get_babel_catalog(), sort_by_file=False, ignore_obsolete=True, include_previous=False)
po_file.seek(0)
# Append the PO file to the archive manifest
archive_files.append( (po_file_path, po_file) )
# Write the MO to a buffer string
mo_file = StringIO()
write_mo(mo_file, catalog.get_babel_catalog(solid=True), use_fuzzy=False)
mo_file.seek(0)
# Append the MO file to the archive manifest
archive_files.append( (mo_file_path, mo_file) )
# Open and fill tarball archive
archive = tarfile.open("{0}.tar.gz".format(project.slug), mode="w:gz", fileobj=archive_fileobj)
mtime = time.time()
# Build the tarball from the manifest
for name,content in archive_files:
info = tarfile.TarInfo(name)
info.size=len(content.getvalue())
info.mtime = mtime
archive.addfile(tarinfo=info, fileobj=content)
#
content.close()
archive.close()
return archive_fileobj
def write_mo(self, locale, warnfunc):
with io.open(self.po_path, 'rt', encoding=self.charset) as file_po:
try:
po = read_po(file_po, locale)
except Exception:
warnfunc('reading error: %s' % self.po_path)
return
with io.open(self.mo_path, 'wb') as file_mo:
try:
write_mo(file_mo, po)
except Exception:
warnfunc('writing error: %s' % self.mo_path)
def test_load(self):
tempdir = tempfile.mkdtemp()
try:
messages_dir = os.path.join(tempdir, 'fr', 'LC_MESSAGES')
os.makedirs(messages_dir)
catalog = Catalog(locale='fr', domain='messages')
catalog.add('foo', 'bar')
write_mo(file(os.path.join(messages_dir, 'messages.mo'), 'wb'), catalog)
translations = support.Translations.load(tempdir, locales=('fr',), domain='messages')
self.assertEqual('bar', translations.gettext('foo'))
finally:
shutil.rmtree(tempdir)
def write_mo(self, locale: str, use_fuzzy: bool = False) -> None:
with open(self.po_path, encoding=self.charset) as file_po:
try:
po = read_po(file_po, locale)
except Exception as exc:
logger.warning(__('reading error: %s, %s'), self.po_path, exc)
return
with open(self.mo_path, 'wb') as file_mo:
try:
write_mo(file_mo, po, use_fuzzy)
except Exception as exc:
logger.warning(__('writing error: %s, %s'), self.mo_path, exc)
def install(self):
generated = []
for dirpath, _, filenames in os.walk(self.podir):
for filename in filenames:
domain, ext = os.path.splitext(filename)
if ext == ".po":
po = open(os.path.join(dirpath, filename))
mo = open(os.path.join(dirpath, domain + ".mo"), "wb")
logging.getLogger(self.name).info("compiling catalog '%s' to '%s'" % (po.name, mo.name))
write_mo(mo, read_po(po))
generated.append(mo.name)
mo.close()
return generated
def setUp(self):
# Use a locale which won't fail to run the tests
os.environ['LANG'] = 'en_US.UTF-8'
messages1 = [
('foo', {
'string': 'Voh'
}),
('foo', {
'string': 'VohCTX',
'context': 'foo'
}),
(('foo1', 'foos1'), {
'string': ('Voh1', 'Vohs1')
}),
(('foo1', 'foos1'), {
'string': ('VohCTX1', 'VohsCTX1'),
'context': 'foo'
}),
]
messages2 = [
('foo', {
'string': 'VohD'
}),
('foo', {
'string': 'VohCTXD',
'context': 'foo'
}),
(('foo1', 'foos1'), {
'string': ('VohD1', 'VohsD1')
}),
(('foo1', 'foos1'), {
'string': ('VohCTXD1', 'VohsCTXD1'),
'context': 'foo'
}),
]
catalog1 = Catalog(locale='en_GB', domain='messages')
catalog2 = Catalog(locale='en_GB', domain='messages1')
for ids, kwargs in messages1:
catalog1.add(ids, **kwargs)
for ids, kwargs in messages2:
catalog2.add(ids, **kwargs)
catalog1_fp = StringIO()
catalog2_fp = StringIO()
write_mo(catalog1_fp, catalog1)
catalog1_fp.seek(0)
write_mo(catalog2_fp, catalog2)
catalog2_fp.seek(0)
translations1 = support.Translations(catalog1_fp)
translations2 = support.Translations(catalog2_fp, domain='messages1')
self.translations = translations1.add(translations2, merge=False)
def write_mo(self, locale):
# type: (str) -> None
with open(self.po_path, encoding=self.charset) as file_po:
try:
po = read_po(file_po, locale)
except Exception as exc:
logger.warning(__('reading error: %s, %s'), self.po_path, exc)
return
with open(self.mo_path, 'wb') as file_mo:
try:
write_mo(file_mo, po)
except Exception as exc:
logger.warning(__('writing error: %s, %s'), self.mo_path, exc)
def test_load(self):
tempdir = tempfile.mkdtemp()
try:
messages_dir = os.path.join(tempdir, 'fr', 'LC_MESSAGES')
os.makedirs(messages_dir)
catalog = Catalog(locale='fr', domain='messages')
catalog.add('foo', 'bar')
with open(os.path.join(messages_dir, 'messages.mo'), 'wb') as f:
write_mo(f, catalog)
translations = support.Translations.load(tempdir, locales=('fr',), domain='messages')
self.assertEqual('bar', translations.gettext('foo'))
finally:
shutil.rmtree(tempdir)
def write_mo(self, locale):
# type: (unicode) -> None
with io.open(self.po_path, 'rt', encoding=self.charset) as file_po:
try:
po = read_po(file_po, locale)
except Exception as exc:
logger.warning(__('reading error: %s, %s'), self.po_path, exc)
return
with io.open(self.mo_path, 'wb') as file_mo:
try:
write_mo(file_mo, po)
except Exception as exc:
logger.warning(__('writing error: %s, %s'), self.mo_path, exc)
def write_mo(self, locale):
# type: (unicode) -> None
with io.open(self.po_path, 'rt', encoding=self.charset) as file_po:
try:
po = read_po(file_po, locale)
except Exception:
logger.warning('reading error: %s', self.po_path)
return
with io.open(self.mo_path, 'wb') as file_mo:
try:
write_mo(file_mo, po)
except Exception:
logger.warning('writing error: %s', self.mo_path)
def _create_catalog(self, path_id, domain=u'foo', locale=u'de', **messages):
i18n_dir = os.path.join(self._tempdir, path_id)
path = os.path.join(i18n_dir, locale, 'LC_MESSAGES')
os.makedirs(path)
mo_filename = os.path.join(path, '%s.mo' % domain)
assert_false(os.path.exists(mo_filename))
catalog = Catalog(locale=locale, domain=domain, fuzzy=False)
for message_id, translation in messages.items():
catalog.add(message_id, translation)
mo_fp = file(mo_filename, 'wb')
write_mo(mo_fp, catalog)
mo_fp.close()
return i18n_dir
def test_load(self):
tempdir = tempfile.mkdtemp()
try:
messages_dir = os.path.join(tempdir, "fr", "LC_MESSAGES")
os.makedirs(messages_dir)
catalog = Catalog(locale="fr", domain="messages")
catalog.add("foo", "bar")
with open(os.path.join(messages_dir, "messages.mo"), "wb") as f:
write_mo(f, catalog)
translations = support.Translations.load(tempdir, locales=("fr",), domain="messages")
self.assertEqual("bar", translations.gettext("foo"))
finally:
shutil.rmtree(tempdir)
def build_translations(self):
"""Create MO files for existing PO translation files."""
for ld in self.languages:
infile = os.path.join(prepare.fetch("l18n"), ld, "LC_MESSAGES",
"base.po")
outfile = os.path.join(os.path.dirname(infile), "base.mo")
# build only complete translations
if os.path.exists(infile) and not os.path.exists(outfile):
with io.open(infile, "r", encoding="UTF8") as po_file:
catalog = read_po(po_file)
with io.open(outfile, "wb") as mo_file:
write_mo(mo_file, catalog)
def _compile(self, app):
for dirpath, dirnames, filenames in filtered_walk(os.path.join(app.path, "locale")):
for filename in filenames:
filename = os.path.join(dirpath, filename)
if filename.endswith(".po"):
with open(filename, "r") as infp:
catalog = pofile.read_po(infp)
if not len(catalog):
continue
sio = StringIO.StringIO()
mofile.write_mo(sio, catalog)
mo_file = filename.replace(".po", ".mo")
with open(mo_file, "wb") as outfp:
outfp.write(sio.getvalue())
self.log.info("%s compiled to %s", filename, mo_file)
def test_catalog_merge_files():
# Refs issues #92, #162
t1 = support.Translations()
assert t1.files == []
t1._catalog["foo"] = "bar"
fp = BytesIO()
write_mo(fp, Catalog())
fp.seek(0)
fp.name = "pro.mo"
t2 = support.Translations(fp)
assert t2.files == ["pro.mo"]
t2._catalog["bar"] = "quux"
t1.merge(t2)
assert t1.files == ["pro.mo"]
assert set(t1._catalog.keys()) == {'', 'foo', 'bar'}
def compile_catalogs(self, languages=None):
"""
Compile PO catalogs to MO files
Note:
Errors have no test coverage since ``read_po()`` pass them through
warnings print to stdout and this is not blocking or detectable.
And so the code continue to the compile part.
Keyword Arguments:
languages (list): List of languages to process. Default is
``None`` so languages are taken from ``LANGUAGES`` settings.
Returns:
list: List of language identifiers for compiled catalogs.
"""
languages = self.parse_languages(languages or self.settings.LANGUAGES)
compiled = []
for locale in languages:
msg = "Compiling catalog (MO) for language '{0}' to {1}"
self.logger.info(msg.format(locale, self.get_mo_filepath(locale)))
with io.open(self.get_po_filepath(locale), 'rb') as fp:
#
catalog = read_po(fp, locale)
# Check errors in catalog
errs = False
for message, errors in catalog.check():
for error in errors:
errs = True
self.logger.warning('Error at line {0}: {1}'.format(
message.lineno,
error
))
# Don't overwrite previous MO file if there have been error
# TODO: Raise exception instead of logging error
if errs:
self.logger.error(("There has been errors within the "
"catalog, compilation has been aborted"))
break
with io.open(self.get_mo_filepath(locale), 'wb') as fp:
write_mo(fp, catalog, use_fuzzy=False)
compiled.append(locale)
return compiled
def write_mo_file(self):
"""
write MO files from existing PO files
"""
print('updating MO file...')
po_path = os.path.join(self.translations_path, self.language,
'LC_MESSAGES', self.translations_key + '.po')
po_file = open(po_path, 'r+')
catalog = read_po(po_file)
mo_path = os.path.join(self.translations_path, self.language,
'LC_MESSAGES', self.translations_key + '.mo')
mo_file = open(mo_path, 'w+')
write_mo(mo_file, catalog)
return
def do_compile_translations(target=("t", ""),
i18n_dir=("i", ""),
all=("a", False)):
"""
Compiling all the templates in specified application.
"""
if not target and not all:
print_status('Please specify target.')
sys.exit(1)
elif target == 'kay':
print_status('Compiling builtin languages')
root = path.join(kay.KAY_DIR, 'i18n')
elif all:
targets = get_user_apps()
for target in targets:
do_compile_translations(target=target, i18n_dir=None, all=False)
do_compile_translations(target=kay.PROJECT_DIR,
i18n_dir=None,
all=False)
sys.exit(0)
else:
if i18n_dir:
root = i18n_dir
else:
root = path.join(target, 'i18n')
if not path.isdir(root):
print('i18n folder missing')
sys.exit(1)
print_status('Compiling %s' % root)
for domain in domains:
for lang in listdir(root):
folder = path.join(root, lang)
translations = path.join(folder, 'LC_MESSAGES', domain + '.po')
if path.isfile(translations):
mo_file = open(translations.replace('.po', '.mo'), 'wb')
print_status('Compiling %s ' % translations)
f = file(translations)
try:
catalog = read_po(f, locale=lang)
finally:
f.close()
# Write standard catalog
write_mo(mo_file, catalog)
mo_file.close()
print_status('All done.')
def _compile(self, app):
for dirpath, dirnames, filenames in filtered_walk(
os.path.join(app.path, "locale")):
for filename in filenames:
filename = os.path.join(dirpath, filename)
print(filename)
if filename.endswith(".po"):
with open(filename, "r") as infp:
catalog = pofile.read_po(infp)
if not len(catalog):
continue
bio = io.BytesIO()
mofile.write_mo(bio, catalog)
mo_file = filename.replace(".po", ".mo")
with open(mo_file, "wb") as outfp:
outfp.write(bio.getvalue())
self.log.info("%s compiled to %s", filename, mo_file)
def install(self):
generated = []
for dirpath, _, filenames in os.walk(self.podir,
followlinks=self.followlinks):
locale = os.path.basename(os.path.dirname(dirpath))
for filename in filenames:
domain, ext = os.path.splitext(filename)
if ext == '.po':
po = open(os.path.join(dirpath, filename))
mo = open(os.path.join(dirpath, domain + '.mo'), 'wb')
logging.getLogger(self.name).info(
"compiling catalog '%s' to '%s'"%(po.name, mo.name))
write_mo(mo, read_po(po, locale))
generated.append(mo.name)
mo.close()
return generated
def compile_catalogs(self, languages=None):
"""
Compile PO catalogs to MO files
Note:
Errors have no test coverage since ``read_po()`` pass them through
warnings print to stdout and this is not blocking or detectable.
And so the code continue to the compile part.
Keyword Arguments:
languages (list): List of languages to process. Default is
``None`` so languages are taken from ``LANGUAGES`` settings.
Returns:
list: List of language identifiers for compiled catalogs.
"""
languages = self.parse_languages(languages or self.settings.LANGUAGES)
compiled = []
for locale in languages:
msg = "Compiling catalog (MO) for language '{0}' to {1}"
self.logger.info(msg.format(locale, self.get_mo_filepath(locale)))
with io.open(self.get_po_filepath(locale), "rb") as fp:
#
catalog = read_po(fp, locale)
# Check errors in catalog
errs = False
for message, errors in catalog.check():
for error in errors:
errs = True
self.logger.warning("Error at line {0}: {1}".format(
message.lineno, error))
# Don't overwrite previous MO file if there have been error
if errs:
self.logger.critical(("There has been errors within the "
"catalog, compilation has been aborted"))
break
with io.open(self.get_mo_filepath(locale), "wb") as fp:
write_mo(fp, catalog, use_fuzzy=False)
compiled.append(locale)
return compiled
def compile_languages(self, remove_po_files=False):
for filename in glob.glob('translations/*/LC_MESSAGES/messages.po'):
language = filename[
len('translations/'):-len('/LC_MESSAGES/messages.po')
]
Logger.info('Translator: Compiling language %s...', language)
with open(filename, 'rb') as po_file:
catalog = pofile.read_po(po_file, locale=language)
mo_filename = filename.replace('.po', '.mo')
with open(mo_filename, 'wb') as mo_file:
mofile.write_mo(mo_file, catalog)
if remove_po_files:
os.remove(filename)
def compile(po_file_path):
"""compile
Compile a xx.po file into an xx.mo file.
:param po_file_path:
"""
dirname = os.path.dirname(po_file_path)
filename = os.path.basename(po_file_path)
locale = os.path.splitext(filename)[0]
mo_file_path = os.path.join(dirname, locale + ".mo")
# Read catalog
with open(po_file_path) as f:
catalog = read_po(f, locale)
# Write .mo file
with open(mo_file_path, "w") as f:
write_mo(f, catalog)
def setUp(self):
self.workspace_dir = tempfile.mkdtemp()
test_locale_dir = os.path.join(self.workspace_dir, 'locales')
mo_path = os.path.join(test_locale_dir, TEST_LANG, 'LC_MESSAGES',
'messages.mo')
os.makedirs(os.path.dirname(mo_path))
# format the test catalog object into MO file format
with open(mo_path, 'wb') as mo_file:
mofile.write_mo(mo_file, TEST_CATALOG)
# patch the LOCALE_DIR variable in natcap/invest/__init__.py
# this is where gettext will look for translations
self.locale_dir_patcher = patch('natcap.invest.LOCALE_DIR',
test_locale_dir)
# patch the LOCALES variable to allow a fake test locale
self.locales_patcher = patch('natcap.invest.LOCALES',
[TEST_LANG, 'en'])
self.locale_dir_patcher.start()
self.locales_patcher.start()
def build_translations(self, recompile_translations=False):
"""Create MO files for existing PO translation files."""
for ld in self.languages:
lang_folder = os.path.join(ld, "LC_MESSAGES")
outfile_folder = os.path.join(paths.CACHE_DIR, "l18n", lang_folder)
infile = os.path.join(prepare.fetch("l18n"), lang_folder,
"base.po")
outfile = os.path.join(outfile_folder, "base.mo")
# build only complete translations
if os.path.exists(infile) and (not os.path.exists(outfile)
or recompile_translations):
with open(infile, "r", encoding="UTF8") as po_file:
catalog = read_po(po_file)
os.makedirs(outfile_folder, exist_ok=True)
with open(outfile, "wb") as mo_file:
write_mo(mo_file, catalog)
logger.debug("writing l18n mo: %s", outfile)
def test_empty_translation_with_fallback(self):
catalog1 = Catalog(locale='fr_FR')
catalog1.add(u'', '''\
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n''')
catalog1.add(u'Fuzz', '')
buf1 = BytesIO()
mofile.write_mo(buf1, catalog1)
buf1.seek(0)
catalog2 = Catalog(locale='fr')
catalog2.add(u'', '''\
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n''')
catalog2.add(u'Fuzz', 'Flou')
buf2 = BytesIO()
mofile.write_mo(buf2, catalog2)
buf2.seek(0)
translations = Translations(fp=buf1)
translations.add_fallback(Translations(fp=buf2))
self.assertEqual(u'Flou', translations.ugettext('Fuzz'))
def compile_message_catalog(lang):
olang = find_language(lang)
if not olang:
print("No language with that name found.")
return
if not os.path.exists(olang.po_file):
print("No po file found for language {0}".format(olang.name))
return
print("Compiling catalog for language {0}".format(olang.name))
print("Reading file {0}".format(olang.po_file))
with open(olang.po_file, 'r') as f:
cat = pofile.read_po(f)
print("Writing file {0}".format(olang.mo_file))
with open(olang.mo_file, 'wb') as f:
mofile.write_mo(f, cat)
print("Done.")
def generate_translations(self):
from StringIO import StringIO
from babel.messages import Catalog, mofile, pofile
from babel.support import Translations
catalog = Catalog(locale='ko_KR')
po = '''
# ugettext
msgid "I like a {0}."
msgstr "나는 {0:을} 좋아합니다.'
# ungettext
msgid "Here is a {0}."
msgid_plural "Here are {1} {0}."
msgstr[0] "여기 {0:이} 있습니다."
msgstr[1] "여기 {0:이} {1}개 있습니다."
# ugettext
msgid "I reached level {0}."
msgstr "나는 레벨{0:이} 되었습니다.'
'''
buf = StringIO()
catalog = pofile.read_po(StringIO(po))
mofile.write_mo(buf, catalog)
buf.seek(0)
return Translations(buf)
def test_catalog_merge_files():
# Refs issues #92, #162
t1 = support.Translations()
assert t1.files == []
t1._catalog["foo"] = "bar"
if PY2:
# Explicitly use the pure-Python `StringIO` class, as we need to
# augment it with the `name` attribute, which we can't do for
# `babel._compat.BytesIO`, which is `cStringIO.StringIO` under
# `PY2`...
from StringIO import StringIO
fp = StringIO()
else:
fp = BytesIO()
write_mo(fp, Catalog())
fp.seek(0)
fp.name = "pro.mo"
t2 = support.Translations(fp)
assert t2.files == ["pro.mo"]
t2._catalog["bar"] = "quux"
t1.merge(t2)
assert t1.files == ["pro.mo"]
assert set(t1._catalog.keys()) == {'', 'foo', 'bar'}
def regenerate_all(self) -> bool:
new_catalog = self._build_strings_catalog_from_sources()
is_complete = True
for language in SUPPORTED_LANGUAGES:
self.locale_dir.joinpath(language,
'LC_MESSAGES').mkdir(parents=True,
exist_ok=True)
updated_catalog = self._get_updated_translations(
language, new_catalog)
is_lang_translations_complete = self._is_translations_complete(
updated_catalog, language)
is_complete = is_complete and is_lang_translations_complete
with self.locale_dir.joinpath(language,
'zordon.po').open(mode='wb') as file:
pofile.write_po(file,
updated_catalog,
sort_output=True,
width=None,
omit_header=True)
with self.locale_dir.joinpath(language, 'LC_MESSAGES',
'zordon.mo').open(mode='wb') as file:
mofile.write_mo(file, updated_catalog)
return is_complete
def compile(self, file_path=None, backup=True, **kwargs):
"""
:param file_path: if the `file_path` attribute is `None`, `path` is
taken as the output file and `file_path` parameter is
discarded. If `file_path` is not `None`, the output
file is `file_path` and `path` is not updated.
Default is `None`. File extension is supposed
to be '.po' and is changed to `.mo` before writing.
:param backup: if `True` and the output file exists, a backup is made
prior to overwrite the file. Further backups overwrite
the previous.
Other optional keyword parameters are passed to
`babel.messages.mofile.write_mo()` see `babel` public API docs.
"""
if file_path is None:
file_path = self.path
if file_path is not None:
file_path = file_path[:-2] + 'mo'
if pth.isfile(file_path) and backup:
shutil.copy(file_path, '{}~'.format(file_path))
with open(file_path, 'wb') as f:
write_mo(f, self.catalog, **kwargs)
def babel_compile_messages(use_fuzzy=False,
statistics=False,
domain=self.cfg.domain): # noqa
"""Compile messages for locales.
:param domain: set domain name for locales
"""
for locales_dir in self.cfg.locale_folders:
for locale in os.listdir(locales_dir):
po_file = os.path.join(locales_dir, locale, 'LC_MESSAGES',
domain + '.po')
if not os.path.exists(po_file):
continue
with open(po_file, 'r') as po:
catalog = read_po(po, locale)
mo_file = os.path.join(locales_dir, locale, 'LC_MESSAGES',
domain + '.mo')
with open(mo_file, 'wb') as mo:
logger.info('writing MO template file to %s', mo_file)
write_mo(mo, catalog, use_fuzzy=use_fuzzy)
def main():
'''
Run the compile code
'''
print('Gathering the translation catalogs to compile...'),
sys.stdout.flush()
entries = {}
for locale in os.listdir(os.path.join(LOCALES_DIR)):
if locale == 'pot':
continue
locale_path = os.path.join(LOCALES_DIR, locale)
entries[locale] = []
for dirpath, _, filenames in os.walk(locale_path):
for filename in fnmatch.filter(filenames, '*.po'):
entries[locale].append(os.path.join(dirpath, filename))
print('DONE')
for locale, po_files in sorted(entries.items()):
lc_messages_path = os.path.join(LOCALES_DIR, locale, 'LC_MESSAGES')
print('\nCompiling the {0!r} locale:'.format(locale))
for po_file in sorted(po_files):
relpath = os.path.relpath(po_file, lc_messages_path)
print ' {0}.po -> {0}.mo'.format(relpath.split('.po', 1)[0])
if HAS_BABEL:
catalog = pofile.read_po(open(po_file))
mofile.write_mo(open(po_file.replace('.po', '.mo'), 'wb'),
catalog)
continue
catalog = polib.pofile(po_file)
catalog.save_as_mofile(fpath=po_file.replace('.po', '.mo'))
print('Done')
