def extract_messages(dirs: list[str]):
catalog = Catalog(project='Open Library',
copyright_holder='Internet Archive')
METHODS = [("**.py", "python"),
("**.html", "openlibrary.i18n:extract_templetor")]
COMMENT_TAGS = ["NOTE:"]
for d in dirs:
extracted = extract_from_dir(d,
METHODS,
comment_tags=COMMENT_TAGS,
strip_comment_tags=True)
counts = {}
for filename, lineno, message, comments, context in extracted:
counts[filename] = counts.get(filename, 0) + 1
catalog.add(message,
None, [(filename, lineno)],
auto_comments=comments)
for filename, count in counts.items():
path = filename if d == filename else os.path.join(d, filename)
print(f"{count}\t{path}", file=sys.stderr)
path = os.path.join(root, 'messages.pot')
f = open(path, 'wb')
write_po(f, catalog)
f.close()
print('wrote template to', path)
def test_checkbox_null_label():
schema_translation = SchemaTranslation()
catalog = Catalog(locale=Locale("cy"))
catalog.add(
"Rugby",
"Rygbi",
)
schema_translation.catalog = catalog
schema = SurveySchema({
"question": {
"answers": [{
"label": None,
"options": [{
"label": "Rugby",
"value": "Rugby"
}]
}],
}
})
translated = schema.translate(schema_translation)
expected = {
"question": {
"answers": [{
"label": None,
"options": [{
"label": "Rygbi",
"value": "Rugby"
}]
}],
},
"language": "cy",
}
assert expected == translated.schema
def test_translation_plugins(app, tmpdir):
session.lang = 'fr_FR'
plugin = MockPlugin(plugin_engine, app)
app.extensions['pluginengine'].plugins['dummy'] = plugin
plugin.root_path = tmpdir.strpath
french_core_str = DICTIONARIES['fr_FR']['This is not a string']
french_plugin_str = "This is not le french string"
trans_dir = os.path.join(plugin.root_path, 'translations', 'fr_FR',
'LC_MESSAGES')
os.makedirs(trans_dir)
# Create proper *.mo file for plugin translation
with open(os.path.join(trans_dir, 'messages.mo'), 'wb') as f:
catalog = Catalog(locale='fr_FR', domain='plugin')
catalog.add("This is not a string", "This is not le french string")
write_mo(f, catalog)
gettext_plugin = make_bound_gettext('dummy')
assert _(u'This is not a string') == french_core_str
assert gettext_context(u"This is not a string") == french_core_str
assert gettext_plugin(u"This is not a string") == french_plugin_str
with plugin.plugin_context():
assert _(u'This is not a string') == french_core_str
assert gettext_context(u"This is not a string") == french_plugin_str
assert gettext_plugin(u"This is not a string") == french_plugin_str
def create_locale(root, locale):
path = os.path.join(root, locale, 'LC_MESSAGES')
os.makedirs(path)
catalog = Catalog(locale=locale)
catalog.add('foo', 'bar {}'.format(locale))
with open(os.path.join(path, 'django.mo'), 'wb') as buf:
write_mo(buf, catalog)
def test_multiple_pos(self):
"""If the language writes to multiple .po files, those are
all counted together. Either all of them will be written,
or none of them will be.
"""
p = self.setup_project()
p.write_xml(kind='strings')
p.write_xml(kind='arrays')
# Create two catalogs, one fully translated, the other one not
# at all.
c = Catalog(locale='de')
c.add('translated', 'value', context='translated')
p.write_po(c, 'strings-de.po')
c = Catalog(locale='de')
c.add('untranslated', context='untranslated')
p.write_po(c, 'arrays-de.po')
# If we require 100% completness on import, both files will
# be empty, even though one of them is fully translated. But
# the second drags down the total of the group.
p.program('import', {'--require-min-complete': '1'})
assert len(p.get_xml('de', kind='strings')) == 0
assert len(p.get_xml('de', kind='arrays')) == 0
def test_translation_plugins(app, tmpdir):
session.lang = 'fr_FR'
plugin = MockPlugin(plugin_engine, app)
app.extensions['pluginengine'].plugins['dummy'] = plugin
plugin.root_path = tmpdir.strpath
french_core_str = DICTIONARIES['fr_FR']['This is not a string']
french_plugin_str = "This is not le french string"
trans_dir = os.path.join(plugin.root_path, 'translations', 'fr_FR', 'LC_MESSAGES')
os.makedirs(trans_dir)
# Create proper *.mo file for plugin translation
with open(os.path.join(trans_dir, 'messages.mo'), 'wb') as f:
catalog = Catalog(locale='fr_FR', domain='plugin')
catalog.add("This is not a string", "This is not le french string")
write_mo(f, catalog)
gettext_plugin = make_bound_gettext('dummy')
assert _(u'This is not a string') == french_core_str
assert gettext_context(u"This is not a string") == french_core_str
assert gettext_plugin(u"This is not a string") == french_plugin_str
with plugin.plugin_context():
assert _(u'This is not a string') == french_core_str
assert gettext_context(u"This is not a string") == french_plugin_str
assert gettext_plugin(u"This is not a string") == french_plugin_str
def test_sorting(self):
# Ensure the header is sorted to the first entry so that its charset
# can be applied to all subsequent messages by GNUTranslations
# (ensuring all messages are safely converted to unicode)
catalog = Catalog(locale='en_US')
catalog.add(
u'', '''\
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n''')
catalog.add(u'foo', 'Voh')
catalog.add((u'There is', u'There are'), (u'Es gibt', u'Es gibt'))
catalog.add(u'Fizz', '')
catalog.add(('Fuzz', 'Fuzzes'), ('', ''))
buf = StringIO()
mofile.write_mo(buf, catalog)
buf.seek(0)
translations = gettext.GNUTranslations(fp=buf)
self.assertEqual(u'Voh', translations.ugettext('foo'))
assert isinstance(translations.ugettext('foo'), unicode)
self.assertEqual(u'Es gibt',
translations.ungettext('There is', 'There are', 1))
assert isinstance(translations.ungettext('There is', 'There are', 1),
unicode)
self.assertEqual(u'Fizz', translations.ugettext('Fizz'))
assert isinstance(translations.ugettext('Fizz'), unicode)
self.assertEqual(u'Fuzz', translations.ugettext('Fuzz'))
assert isinstance(translations.ugettext('Fuzz'), unicode)
self.assertEqual(u'Fuzzes', translations.ugettext('Fuzzes'))
assert isinstance(translations.ugettext('Fuzzes'), unicode)
def test_sorting(self):
# Ensure the header is sorted to the first entry so that its charset
# can be applied to all subsequent messages by GNUTranslations
# (ensuring all messages are safely converted to unicode)
catalog = Catalog(locale='en_US')
catalog.add(u'', '''\
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n''')
catalog.add(u'foo', 'Voh')
catalog.add((u'There is', u'There are'), (u'Es gibt', u'Es gibt'))
catalog.add(u'Fizz', '')
catalog.add(('Fuzz', 'Fuzzes'), ('', ''))
buf = StringIO()
mofile.write_mo(buf, catalog)
buf.seek(0)
translations = gettext.GNUTranslations(fp=buf)
self.assertEqual(u'Voh', translations.ugettext('foo'))
assert isinstance(translations.ugettext('foo'), unicode)
self.assertEqual(u'Es gibt', translations.ungettext('There is', 'There are', 1))
assert isinstance(translations.ungettext('There is', 'There are', 1), unicode)
self.assertEqual(u'Fizz', translations.ugettext('Fizz'))
assert isinstance(translations.ugettext('Fizz'), unicode)
self.assertEqual(u'Fuzz', translations.ugettext('Fuzz'))
assert isinstance(translations.ugettext('Fuzz'), unicode)
self.assertEqual(u'Fuzzes', translations.ugettext('Fuzzes'))
assert isinstance(translations.ugettext('Fuzzes'), unicode)
def test_variant_translation(schema_with_question_variants):
schema_translation = SchemaTranslation()
catalog = Catalog()
catalog.add(
"First name",
"WELSH - First name",
context="What is your name?",
)
catalog.add(
"First name",
"WELSH - First name - Proxy",
context="What is their name?",
)
schema_translation.catalog = catalog
variant_schema = SurveySchema(schema_with_question_variants)
translated = variant_schema.translate(schema_translation)
assert (translated.schema["question_variants"][0]["question"]["answers"][0]
["label"] == "WELSH - First name")
assert (translated.schema["question_variants"][1]["question"]["answers"][0]
["label"] == "WELSH - First name - Proxy")
def test_multiple_pos(self):
"""If the language writes to multiple .po files, those are
all counted together. Either all of them will be written,
or none of them will be.
"""
p = self.setup_project()
p.write_xml(kind='strings')
p.write_xml(kind='arrays')
# Create two catalogs, one fully translated, the other one not
# at all.
c = Catalog(locale='de')
c.add('translated', 'value', context='translated')
p.write_po(c, 'strings-de.po')
c = Catalog(locale='de')
c.add('untranslated', context='untranslated')
p.write_po(c, 'arrays-de.po')
# If we require 100% completness on import, both files will
# be empty, even though one of them is fully translated. But
# the second drags down the total of the group.
p.program('import', {'--require-min-complete': '1'})
assert len(p.get_xml('de', kind='strings')) == 0
assert len(p.get_xml('de', kind='arrays')) == 0
def assert_convert(cls, po, xml=None, namespaces={}):
"""Helper that passes the string in ``po`` through our po
to xml converter, and checks the resulting xml string value
against ``xml``.
If ``xml`` is not given, we check against ``po`` instead, i.e.
expect the string to remain unchanged.
"""
key = 'test'
catalog = Catalog()
catalog.add(po, po, context=key)
dom = po2xml(catalog)
elem = dom.xpath('/resources/string[@name="%s"]' % key)[0]
elem_as_text = etree.tostring(elem, encoding=unicode)
value = re.match("^[^>]+>(.*)<[^<]+$", elem_as_text).groups(1)[0]
match = xml if xml is not None else po
print "'%s' == '%s'" % (value, match)
print repr(value), '==', repr(match)
assert value == match
# If ``namespaces`` are set, the test expects those to be defined
# in the root of the document
for prefix, uri in namespaces.items():
assert dom.nsmap[prefix] == uri
# In this case, the reverse (converting back to po) always needs
# to give us the original again, so this allows for a nice extra
# check.
TestFromXML.assert_convert(match, po, namespaces)
def setUp(self):
# Use a locale which won't fail to run the tests
os.environ["LANG"] = "en_US.UTF-8"
messages1 = [
("foo", {"string": "Voh"}),
("foo", {"string": "VohCTX", "context": "foo"}),
(("foo1", "foos1"), {"string": ("Voh1", "Vohs1")}),
(("foo1", "foos1"), {"string": ("VohCTX1", "VohsCTX1"), "context": "foo"}),
]
messages2 = [
("foo", {"string": "VohD"}),
("foo", {"string": "VohCTXD", "context": "foo"}),
(("foo1", "foos1"), {"string": ("VohD1", "VohsD1")}),
(("foo1", "foos1"), {"string": ("VohCTXD1", "VohsCTXD1"), "context": "foo"}),
]
catalog1 = Catalog(locale="en_GB", domain="messages")
catalog2 = Catalog(locale="en_GB", domain="messages1")
for ids, kwargs in messages1:
catalog1.add(ids, **kwargs)
for ids, kwargs in messages2:
catalog2.add(ids, **kwargs)
catalog1_fp = BytesIO()
catalog2_fp = BytesIO()
write_mo(catalog1_fp, catalog1)
catalog1_fp.seek(0)
write_mo(catalog2_fp, catalog2)
catalog2_fp.seek(0)
translations1 = support.Translations(catalog1_fp)
translations2 = support.Translations(catalog2_fp, domain="messages1")
self.translations = translations1.add(translations2, merge=False)
def _extract(self, app):
catalog = Catalog(domain="django", charset="utf8")
files = {}
for dirpath, dirnames, filenames in filtered_walk(app.path):
for filename in filenames:
filename = os.path.join(dirpath, filename)
if ACCEPTABLE_FILENAMES_RE.match(filename):
rel_filename = filename[len(os.path.commonprefix((app.path, filename))) + 1:].replace(os.sep, "/")
files[rel_filename] = filename
self.log.info("%s: %d translatable files found", app.label, len(files))
extractors = self.get_extractors()
for rel_filename, filename in sorted(files.items()):
extractor_tup = extractors.get(os.path.splitext(filename)[1])
if not extractor_tup:
self.log.warning("Not sure how to extract messages from %s", filename)
continue
extractor, options = extractor_tup
with open(filename, "rb") as fp:
for (lineno, message, comments, context) in extract(extractor, fp, options=options):
catalog.add(message, locations=[(rel_filename, 0)], auto_comments=comments)
if len(catalog):
pot_path = self._get_pot_path(app)
with open(pot_path, "w") as outf:
pofile.write_po(outf, catalog, width=1000, omit_header=True, sort_output=True)
self.log.info("%s: %d messages in %s", app.label, len(catalog), pot_path)
return catalog
def convert_xlsm_content(contents):
catalog = Catalog()
for content in contents:
resid = content[0]
res_org_content = content[1]
rescontent = content[2]
if resid == None:
continue
if rescontent == None:
rescontent = u""
if match(r"\((.+)(::(.+))+\)", res_org_content):
msg_id = res_org_content[1:-1].split("::")
msg_string = rescontent[1:-1].split("::")
catalog.add(msg_id, context=resid, string=msg_string)
else:
if match(r"\".+\"", res_org_content):
res_org_content = res_org_content[1:-1]
if match(r"\".+\"", rescontent):
rescontent = rescontent[1:-1]
catalog.add(res_org_content, context=resid, string=rescontent)
return catalog
def convert_xlsm_content(contents):
catalog = Catalog()
for content in contents:
resid = content[0]
res_org_content = content[1]
rescontent = content[2]
if resid == None:
continue
if rescontent == None:
rescontent = u""
if match(r"\((.+)(::(.+))+\)", res_org_content):
msg_id = res_org_content[1:-1].split("::")
msg_string = rescontent[1:-1].split("::")
catalog.add(msg_id, context=resid, string=msg_string)
else:
if match(r"\".+\"", res_org_content):
res_org_content = res_org_content[1:-1]
if match(r"\".+\"", rescontent):
rescontent = rescontent[1:-1]
catalog.add(res_org_content, context=resid, string=rescontent)
return catalog
def extract_messages(dirs):
catalog = Catalog(project='Open Library',
copyright_holder='Internet Archive')
METHODS = [("**.py", "python"),
("**.html", "openlibrary.i18n:extract_templetor")]
COMMENT_TAGS = ["NOTE:"]
for d in dirs:
if '.html' in d:
extracted = [(d, ) + extract for extract in extract_from_file(
"openlibrary.i18n:extract_templetor", d)]
else:
extracted = extract_from_dir(d,
METHODS,
comment_tags=COMMENT_TAGS,
strip_comment_tags=True)
for filename, lineno, message, comments, context in extracted:
catalog.add(message,
None, [(filename, lineno)],
auto_comments=comments)
path = os.path.join(root, 'messages.pot')
f = open(path, 'w')
write_po(f, catalog)
f.close()
print('wrote template to', path)
def setUp(self):
# Use a locale which won't fail to run the tests
os.environ['LANG'] = 'en_US.UTF-8'
messages1 = [
('foo', {'string': 'Voh'}),
('foo', {'string': 'VohCTX', 'context': 'foo'}),
(('foo1', 'foos1'), {'string': ('Voh1', 'Vohs1')}),
(('foo1', 'foos1'), {'string': ('VohCTX1', 'VohsCTX1'), 'context': 'foo'}),
]
messages2 = [
('foo', {'string': 'VohD'}),
('foo', {'string': 'VohCTXD', 'context': 'foo'}),
(('foo1', 'foos1'), {'string': ('VohD1', 'VohsD1')}),
(('foo1', 'foos1'), {'string': ('VohCTXD1', 'VohsCTXD1'), 'context': 'foo'}),
]
catalog1 = Catalog(locale='en_GB', domain='messages')
catalog2 = Catalog(locale='en_GB', domain='messages1')
for ids, kwargs in messages1:
catalog1.add(ids, **kwargs)
for ids, kwargs in messages2:
catalog2.add(ids, **kwargs)
catalog1_fp = BytesIO()
catalog2_fp = BytesIO()
write_mo(catalog1_fp, catalog1)
catalog1_fp.seek(0)
write_mo(catalog2_fp, catalog2)
catalog2_fp.seek(0)
translations1 = support.Translations(catalog1_fp)
translations2 = support.Translations(catalog2_fp, domain='messages1')
self.translations = translations1.add(translations2, merge=False)
def assert_convert(cls, po, xml=None, namespaces={}):
"""Helper that passes the string in ``po`` through our po
to xml converter, and checks the resulting xml string value
against ``xml``.
If ``xml`` is not given, we check against ``po`` instead, i.e.
expect the string to remain unchanged.
"""
key = 'test'
catalog = Catalog()
catalog.add(po, po, context=key)
warnfunc = TestWarnFunc()
dom = write_xml(po2xml(catalog, warnfunc=warnfunc), warnfunc=warnfunc)
elem = dom.xpath('/resources/string[@name="%s"]' % key)[0]
elem_as_text = etree.tostring(elem, encoding=unicode)
value = re.match("^[^>]+>(.*)<[^<]+$", elem_as_text).groups(1)[0]
match = xml if xml is not None else po
print "'%s' == '%s'" % (value, match)
print repr(value), '==', repr(match)
assert value == match
# If ``namespaces`` are set, the test expects those to be defined
# in the root of the document
for prefix, uri in namespaces.items():
assert dom.nsmap[prefix] == uri
# In this case, the reverse (converting back to po) always needs to
# give us the original again, so this allows for a nice extra check.
if not match: # Skip this if we are doing a special, custom match.
TestFromXML.assert_convert(match, po, namespaces)
return warnfunc
def extract_messages(dirs):
catalog = Catalog(
project='Open Library',
copyright_holder='Internet Archive'
)
METHODS = [
("**.py", "python"),
("**.html", "openlibrary.i18n:extract_templetor")
]
COMMENT_TAGS = ["NOTE:"]
for d in dirs:
if '.html' in d:
extracted = [(d,) + extract for extract in extract_from_file("openlibrary.i18n:extract_templetor", d)]
else:
extracted = extract_from_dir(d, METHODS, comment_tags=COMMENT_TAGS, strip_comment_tags=True)
for filename, lineno, message, comments, context in extracted:
catalog.add(message, None, [(filename, lineno)], auto_comments=comments)
path = os.path.join(root, 'messages.pot')
f = open(path, 'w')
write_po(f, catalog)
f.close()
print('wrote template to', path)
def test_sorting(self):
# Ensure the header is sorted to the first entry so that its charset
# can be applied to all subsequent messages by GNUTranslations
# (ensuring all messages are safely converted to unicode)
catalog = Catalog(locale="en_US")
catalog.add(
u"",
"""\
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n""",
)
catalog.add(u"foo", "Voh")
catalog.add((u"There is", u"There are"), (u"Es gibt", u"Es gibt"))
catalog.add(u"Fizz", "")
catalog.add(("Fuzz", "Fuzzes"), ("", ""))
buf = StringIO()
mofile.write_mo(buf, catalog)
buf.seek(0)
translations = gettext.GNUTranslations(fp=buf)
self.assertEqual(u"Voh", translations.ugettext("foo"))
assert isinstance(translations.ugettext("foo"), unicode)
self.assertEqual(u"Es gibt", translations.ungettext("There is", "There are", 1))
assert isinstance(translations.ungettext("There is", "There are", 1), unicode)
self.assertEqual(u"Fizz", translations.ugettext("Fizz"))
assert isinstance(translations.ugettext("Fizz"), unicode)
self.assertEqual(u"Fuzz", translations.ugettext("Fuzz"))
assert isinstance(translations.ugettext("Fuzz"), unicode)
self.assertEqual(u"Fuzzes", translations.ugettext("Fuzzes"))
assert isinstance(translations.ugettext("Fuzzes"), unicode)
def setUp(self):
# Use a locale which won't fail to run the tests
os.environ['LANG'] = 'en_US.UTF-8'
messages1 = [
('foo', {'string': 'Voh'}),
('foo', {'string': 'VohCTX', 'context': 'foo'}),
(('foo1', 'foos1'), {'string': ('Voh1', 'Vohs1')}),
(('foo1', 'foos1'), {'string': ('VohCTX1', 'VohsCTX1'), 'context': 'foo'}),
]
messages2 = [
('foo', {'string': 'VohD'}),
('foo', {'string': 'VohCTXD', 'context': 'foo'}),
(('foo1', 'foos1'), {'string': ('VohD1', 'VohsD1')}),
(('foo1', 'foos1'), {'string': ('VohCTXD1', 'VohsCTXD1'), 'context': 'foo'}),
]
catalog1 = Catalog(locale='en_GB', domain='messages')
catalog2 = Catalog(locale='en_GB', domain='messages1')
for ids, kwargs in messages1:
catalog1.add(ids, **kwargs)
for ids, kwargs in messages2:
catalog2.add(ids, **kwargs)
catalog1_fp = BytesIO()
catalog2_fp = BytesIO()
write_mo(catalog1_fp, catalog1)
catalog1_fp.seek(0)
write_mo(catalog2_fp, catalog2)
catalog2_fp.seek(0)
translations1 = support.Translations(catalog1_fp)
translations2 = support.Translations(catalog2_fp, domain='messages1')
self.translations = translations1.add(translations2, merge=False)
def mkcatalog(locale='de'):
"""Helper that returns a gettext catalog with one message
already added.
Tests can add a broken message and then ensure that at least
the valid message still was processed.
"""
c = Catalog(locale='de')
c.add('valid_message', 'valid_value', context='valid_message')
return c
def mkcatalog(locale='de'):
"""Helper that returns a gettext catalog with one message
already added.
Tests can add a broken message and then ensure that at least
the valid message still was processed.
"""
c = Catalog(locale='de')
c.add('valid_message', 'valid_value', context='valid_message')
return c
def test(self):
p = self.setup_project()
c = Catalog(locale='de')
c.add('en1', 'de1', flags=('fuzzy',), context='foo')
c.add('en2', 'de2', context='bar')
p.write_po(c, 'de.po')
p.program('import', {'--ignore-fuzzy': True})
xml = p.get_xml('de')
assert not 'foo' in xml
assert 'bar' in xml
def test(self):
p = self.setup_project()
c = Catalog(locale='de')
c.add('en1', 'de1', flags=('fuzzy', ), context='foo')
c.add('en2', 'de2', context='bar')
p.write_po(c, 'de.po')
p.program('import', {'--ignore-fuzzy': True})
xml = p.get_xml('de')
assert not 'foo' in xml
assert 'bar' in xml
def get_translation_catalog_from_index(index, project_name):
# Get all Modulemd.Module object names
module_names = index.get_module_names()
# Create a list containing highest version streams of a module
final_streams = list()
for module_name in module_names:
module = index.get_module(module_name)
stream_names = module.get_stream_names()
for stream_name in stream_names:
# The first item returned is guaranteed to be the highest version
# of that stream in that module.
stream = module.search_streams(stream_name, 0)[0]
final_streams.append(stream)
# A dictionary to store:
# key: all translatable strings
# value: their respective locations
translation_dict = defaultdict(list)
for stream in final_streams:
# Process description
description = stream.get_description("C")
if description is not None:
location = (("{};{};description").format(
stream.props.module_name, stream.props.stream_name), 2)
translation_dict[description].append(location)
# Process summary
summary = stream.get_summary("C")
if summary is not None:
location = (("{};{};summary").format(
stream.props.module_name, stream.props.stream_name), 1)
translation_dict[summary].append(location)
# Process profile descriptions(sometimes NULL)
profile_names = stream.get_profile_names()
if(profile_names):
for pro_name in profile_names:
profile = stream.get_profile(pro_name)
profile_desc = profile.get_description("C")
if profile_desc is not None:
location = (("{};{};profile;{}").format(
stream.props.module_name, stream.props.stream_name, pro_name), 3)
translation_dict[profile_desc].append(location)
catalog = Catalog(project=project_name)
for translatable_string, locations in translation_dict.items():
logging.debug("This is a translatable_string: %s" % translatable_string)
catalog.add(translatable_string, locations=locations)
return catalog
def test_invalid_xhtml():
"""Ensure we can deal with broken XML in messages.
"""
c = Catalog()
c.add('Foo', '<i>Tag is not closed', context="foo")
# [bug] This caused an exception in 16263b.
dom = write_xml(po2xml(c))
# The tag was closed automatically (our loose parser tries to fix errors).
assert etree.tostring(dom) == b'<resources><string name="foo"><i>Tag is not closed</i></string></resources>'
def test_translate_sets_language():
catalog = Catalog(locale=Locale("cy"))
catalog.add("test")
schema_translation = SchemaTranslation(catalog=catalog)
schema = SurveySchema({})
schema.translate(schema_translation)
translated_schema = schema.translate(schema_translation)
assert translated_schema.language == "cy"
def test_invalid_xhtml():
"""Ensure we can deal with broken XML in messages.
"""
c = Catalog()
c.add('Foo', '<i>Tag is not closed', context="foo")
# [bug] This caused an exception in 16263b.
dom = write_xml(po2xml(c))
# The tag was closed automatically (our loose parser tries to fix errors).
assert etree.tostring(
dom
) == b'<resources><string name="foo"><i>Tag is not closed</i></string></resources>'
def test_load(self):
tempdir = tempfile.mkdtemp()
try:
messages_dir = os.path.join(tempdir, 'fr', 'LC_MESSAGES')
os.makedirs(messages_dir)
catalog = Catalog(locale='fr', domain='messages')
catalog.add('foo', 'bar')
write_mo(file(os.path.join(messages_dir, 'messages.mo'), 'wb'), catalog)
translations = support.Translations.load(tempdir, locales=('fr',), domain='messages')
self.assertEqual('bar', translations.gettext('foo'))
finally:
shutil.rmtree(tempdir)
def test_untranslated():
"""Test that by default, untranslated strings are not included in the
imported XML.
"""
catalog = Catalog()
catalog.add('green', context='color1')
catalog.add('red', 'rot', context='color2')
assert po2xml(catalog) == {'color2': 'rot'}
# If with_untranslated is passed, then all strings are included.
# Note that arrays behave differently (they always include all
# strings), and this is tested in test_string_arrays.py).
assert po2xml(catalog, with_untranslated=True) ==\
{'color1': 'green', 'color2': 'rot'}
def test_untranslated():
"""Test that by default, untranslated strings are not included in the
imported XML.
"""
catalog = Catalog()
catalog.add('green', context='color1')
catalog.add('red', 'rot', context='color2')
assert po2xml(catalog) == {'color2': 'rot'}
# If with_untranslated is passed, then all strings are included.
# Note that arrays behave differently (they always include all
# strings), and this is tested in test_string_arrays.py).
assert po2xml(catalog, with_untranslated=True) ==\
{'color1': 'green', 'color2': 'rot'}
def test_load(self):
tempdir = tempfile.mkdtemp()
try:
messages_dir = os.path.join(tempdir, 'fr', 'LC_MESSAGES')
os.makedirs(messages_dir)
catalog = Catalog(locale='fr', domain='messages')
catalog.add('foo', 'bar')
with open(os.path.join(messages_dir, 'messages.mo'), 'wb') as f:
write_mo(f, catalog)
translations = support.Translations.load(tempdir, locales=('fr',), domain='messages')
self.assertEqual('bar', translations.gettext('foo'))
finally:
shutil.rmtree(tempdir)
def _create_catalog(self, path_id, domain=u'foo', locale=u'de', **messages):
i18n_dir = os.path.join(self._tempdir, path_id)
path = os.path.join(i18n_dir, locale, 'LC_MESSAGES')
os.makedirs(path)
mo_filename = os.path.join(path, '%s.mo' % domain)
assert_false(os.path.exists(mo_filename))
catalog = Catalog(locale=locale, domain=domain, fuzzy=False)
for message_id, translation in messages.items():
catalog.add(message_id, translation)
mo_fp = file(mo_filename, 'wb')
write_mo(mo_fp, catalog)
mo_fp.close()
return i18n_dir
def test_load(self):
tempdir = tempfile.mkdtemp()
try:
messages_dir = os.path.join(tempdir, "fr", "LC_MESSAGES")
os.makedirs(messages_dir)
catalog = Catalog(locale="fr", domain="messages")
catalog.add("foo", "bar")
with open(os.path.join(messages_dir, "messages.mo"), "wb") as f:
write_mo(f, catalog)
translations = support.Translations.load(tempdir, locales=("fr",), domain="messages")
self.assertEqual("bar", translations.gettext("foo"))
finally:
shutil.rmtree(tempdir)
def test_clear(self):
"""Explicitely test that if we ignore a language, the xml
file is overwritten with an empty version. Just not processing
it is not enough.
"""
p = self.setup_project()
# We start out with one string in the XML
p.write_xml({'string1': 'value1'}, lang='de')
assert len(p.get_xml('de')) == 1
c = Catalog(locale='de')
c.add('string1', context='string1')
p.write_po(c, 'de.po')
# Now after the import, the resource file is empty.
p.program('import', {'--require-min-complete': '1'})
assert len(p.get_xml('de')) == 0
def test_clear(self):
"""Explicitely test that if we ignore a language, the xml
file is overwritten with an empty version. Just not processing
it is not enough.
"""
p = self.setup_project()
# We start out with one string in the XML
p.write_xml({'string1': 'value1'}, lang='de')
assert len(p.get_xml('de')) == 1
c = Catalog(locale='de')
c.add('string1', context='string1')
p.write_po(c, 'de.po')
# Now after the import, the resource file is empty.
p.program('import', {'--require-min-complete': '1'})
assert len(p.get_xml('de')) == 0
def test_fuzzy(self):
"""This option is affected by the --ignore-fuzzy flag. If
it is set, fuzzy strings are not counted towards the total.
"""
p = self.setup_project()
c = Catalog(locale='de')
c.add('translated', 'value', context='translated')
c.add('fuzzy', 'value', context='fuzzy', flags=('fuzzy',))
p.write_po(c, 'de.po')
# When fuzzy strings are counted, the catalog above is 100%
# complete.
p.program('import', {'--require-min-complete': '1'})
assert len(p.get_xml('de')) == 2
# If they aren't, it won't make the cut and the result should
# be no strings at all being written.
p.program('import', {'--require-min-complete': '1',
'--ignore-fuzzy': True})
assert len(p.get_xml('de')) == 0
def test_update(self):
template = Catalog()
template.add("1")
template.add("2")
template.add("3")
tmpl_file = os.path.join(self._i18n_dir(), 'temp-template.pot')
with open(tmpl_file, "wb") as outfp:
write_po(outfp, template)
po_file = os.path.join(self._i18n_dir(), 'temp1.po')
self.cli.run(sys.argv + ['init',
'-l', 'fi',
'-o', po_file,
'-i', tmpl_file
])
with open(po_file, "r") as infp:
catalog = read_po(infp)
assert len(catalog) == 3
# Add another entry to the template
template.add("4")
with open(tmpl_file, "wb") as outfp:
write_po(outfp, template)
self.cli.run(sys.argv + ['update',
'-l', 'fi_FI',
'-o', po_file,
'-i', tmpl_file])
with open(po_file, "r") as infp:
catalog = read_po(infp)
assert len(catalog) == 4 # Catalog was updated
def test_update(self):
template = Catalog()
template.add("1")
template.add("2")
template.add("3")
tmpl_file = os.path.join(self._i18n_dir(), "temp-template.pot")
with open(tmpl_file, "wb") as outfp:
write_po(outfp, template)
po_file = os.path.join(self._i18n_dir(), "temp1.po")
self.cli.run(sys.argv + ["init", "-l", "fi", "-o", po_file, "-i", tmpl_file])
with open(po_file, "r") as infp:
catalog = read_po(infp)
assert len(catalog) == 3
# Add another entry to the template
template.add("4")
with open(tmpl_file, "wb") as outfp:
write_po(outfp, template)
self.cli.run(sys.argv + ["update", "-l", "fi_FI", "-o", po_file, "-i", tmpl_file])
with open(po_file, "r") as infp:
catalog = read_po(infp)
assert len(catalog) == 4 # Catalog was updated
def test_update(self):
template = Catalog()
template.add("1")
template.add("2")
template.add("3")
tmpl_file = os.path.join(self._i18n_dir(), 'temp-template.pot')
with open(tmpl_file, "wb") as outfp:
write_po(outfp, template)
po_file = os.path.join(self._i18n_dir(), 'temp1.po')
self.cli.run(sys.argv +
['init', '-l', 'fi', '-o', po_file, '-i', tmpl_file])
with open(po_file, "r") as infp:
catalog = read_po(infp)
assert len(catalog) == 3
# Add another entry to the template
template.add("4")
with open(tmpl_file, "wb") as outfp:
write_po(outfp, template)
self.cli.run(sys.argv +
['update', '-l', 'fi_FI', '-o', po_file, '-i', tmpl_file])
with open(po_file, "r") as infp:
catalog = read_po(infp)
assert len(catalog) == 4 # Catalog was updated
def _extract(self, app):
catalog = Catalog(domain="django", charset="utf8")
files = {}
for dirpath, dirnames, filenames in filtered_walk(app.path):
for filename in filenames:
filename = os.path.join(dirpath, filename)
if ACCEPTABLE_FILENAMES_RE.match(filename):
rel_filename = filename[
len(os.path.commonprefix((app.path, filename))) +
1:].replace(os.sep, "/")
files[rel_filename] = filename
self.log.info("%s: %d translatable files found", app.label, len(files))
extractors = self.get_extractors()
for rel_filename, filename in sorted(files.items()):
extractor_tup = extractors.get(os.path.splitext(filename)[1])
if not extractor_tup:
self.log.warning("Not sure how to extract messages from %s",
filename)
continue
extractor, options = extractor_tup
with open(filename, "rb") as fp:
for (lineno, message, comments,
context) in extract(extractor, fp, options=options):
catalog.add(message,
locations=[(rel_filename, 0)],
auto_comments=comments)
if len(catalog):
pot_path = _get_pot_path(app)
with open(pot_path, "w") as outf:
pofile.write_po(outf,
catalog,
width=1000,
omit_header=True,
sort_output=True)
self.log.info("%s: %d messages in %s", app.label, len(catalog),
pot_path)
return catalog
def test_fuzzy(self):
"""This option is affected by the --ignore-fuzzy flag. If
it is set, fuzzy strings are not counted towards the total.
"""
p = self.setup_project()
c = Catalog(locale='de')
c.add('translated', 'value', context='translated')
c.add('fuzzy', 'value', context='fuzzy', flags=('fuzzy', ))
p.write_po(c, 'de.po')
# When fuzzy strings are counted, the catalog above is 100%
# complete.
p.program('import', {'--require-min-complete': '1'})
assert len(p.get_xml('de')) == 2
# If they aren't, it won't make the cut and the result should
# be no strings at all being written.
p.program('import', {
'--require-min-complete': '1',
'--ignore-fuzzy': True
})
assert len(p.get_xml('de')) == 0
def main():
print "Extracting messages"
root = path.abspath(path.join(path.dirname(__file__), '..'))
catalog = Catalog(msgid_bugs_address=BUGS_ADDRESS,
copyright_holder=COPYRIGHT,
charset="utf-8")
def callback(filename, method, options):
if method != "ignore":
print strip_path(filename, root)
extracted_py = extract_from_dir(root, PY_METHODS, {}, PY_KEYWORDS,
COMMENT_TAGS, callback=callback,
strip_comment_tags=True)
extracted_js = extract_from_dir(root, [("static/js/**.js", "javascript")],
{}, JS_KEYWORDS,
COMMENT_TAGS, callback=callback,
strip_comment_tags=True)
for filename, lineno, message, comments in chain(extracted_py,
extracted_js):
catalog.add(message, None, [(strip_path(filename, root), lineno)],
auto_comments=comments)
output_path = path.join(root, "i18n")
if not path.isdir(output_path):
makedirs(output_path)
f = file(path.join(output_path, "messages.pot"), "w")
try:
write_po(f, catalog, width=79)
finally:
f.close()
print "All done."
def test_empty_translation_with_fallback(self):
catalog1 = Catalog(locale='fr_FR')
catalog1.add(u'', '''\
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n''')
catalog1.add(u'Fuzz', '')
buf1 = BytesIO()
mofile.write_mo(buf1, catalog1)
buf1.seek(0)
catalog2 = Catalog(locale='fr')
catalog2.add(u'', '''\
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n''')
catalog2.add(u'Fuzz', 'Flou')
buf2 = BytesIO()
mofile.write_mo(buf2, catalog2)
buf2.seek(0)
translations = Translations(fp=buf1)
translations.add_fallback(Translations(fp=buf2))
self.assertEqual(u'Flou', translations.ugettext('Fuzz'))
def test_empty_translation_with_fallback(self):
catalog1 = Catalog(locale='fr_FR')
catalog1.add(u'', '''\
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n''')
catalog1.add(u'Fuzz', '')
buf1 = BytesIO()
mofile.write_mo(buf1, catalog1)
buf1.seek(0)
catalog2 = Catalog(locale='fr')
catalog2.add(u'', '''\
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n''')
catalog2.add(u'Fuzz', 'Flou')
buf2 = BytesIO()
mofile.write_mo(buf2, catalog2)
buf2.seek(0)
translations = Translations(fp=buf1)
translations.add_fallback(Translations(fp=buf2))
self.assertEqual(u'Flou', translations.ugettext('Fuzz'))
def test(self):
p = self.setup_project()
c = Catalog(locale='de')
c.add('translated', 'value', context='translated')
c.add('missing1', context='missing1')
c.add('missing2', context='missing2')
c.add('missing3', context='missing3')
p.write_po(c, 'de.po')
# At first, we require half the strings to be available.
# This is clearly not the case in the catalog above.
p.program('import', {'--require-min-complete': '0.5'})
assert len(p.get_xml('de')) == 0
# Now, only require 25% - this should just make the cut.
p.program('import', {'--require-min-complete': '0.25'})
assert len(p.get_xml('de')) == 1
def test(self):
p = self.setup_project()
c = Catalog(locale='de')
c.add('translated', 'value', context='translated')
c.add('missing1', context='missing1')
c.add('missing2', context='missing2')
c.add('missing3', context='missing3')
p.write_po(c, 'de.po')
# At first, we require half the strings to be available.
# This is clearly not the case in the catalog above.
p.program('import', {'--require-min-complete': '0.5'})
assert len(p.get_xml('de')) == 0
# Now, only require 25% - this should just make the cut.
p.program('import', {'--require-min-complete': '0.25'})
assert len(p.get_xml('de')) == 1
def xml2po(file, translations=None, filter=None, warnfunc=dummy_warn):
"""Return the Android string resource in ``file`` as a babel
.po catalog.
If given, the Android string resource in ``translations`` will be
used for the translated values. In this case, the returned value
is a 2-tuple (catalog, unmatched), with the latter being a list of
Android string resource names that are in the translated file, but
not in the original.
Both arguments may also be an already loaded dict of xml strings,
as returned by ``read_xml``.
"""
original_strings = file if isinstance(file, dict) else read_xml(file, warnfunc)
trans_strings = None
if translations is not None:
trans_strings = translations \
if isinstance(translations, dict) \
else read_xml(translations, warnfunc)
catalog = Catalog()
for name, org_value in original_strings.iteritems():
if filter and filter(name):
continue
trans_value = None
if trans_strings:
trans_value = trans_strings.pop(name, trans_value)
if isinstance(org_value, list):
# a string-array, write as "name:index"
if len(org_value) == 0:
warnfunc("Warning: string-array '%s' is empty" % name, 'warning')
continue
if trans_value and not isinstance(trans_value, list):
warnfunc(('""%s" is a string-array in the reference file, '
'but not in the translation.') % name, 'warning')
# makes further processing easier if we can assume
# this is a list
trans_value = []
elif trans_value is None:
trans_value = []
for index, item in enumerate(org_value):
item_trans = trans_value[index].text if index < len(trans_value) else u''
# If the string has formatting markers, indicate it in the gettext output
flags = []
if item.formatted:
flags.append('c-format')
ctx = "%s:%d" % (name, index)
catalog.add(item.text, item_trans, auto_comments=item.comments,
flags=flags, context=ctx)
else:
# a normal string
flags = []
# If the string has formatting markers, indicate it in the gettext output
if org_value.formatted:
flags.append('c-format')
catalog.add(org_value.text, trans_value.text if trans_value else u'',
flags=flags, auto_comments=org_value.comments, context=name)
if trans_strings is not None:
# At this point, trans_strings only contains those for which
# no original existed.
return catalog, trans_strings.keys()
else:
return catalog
def test_more_plural_forms(self):
catalog2 = Catalog(locale='ru_RU')
catalog2.add(('Fuzz', 'Fuzzes'), ('', '', ''))
buf = StringIO()
mofile.write_mo(buf, catalog2)
def xml2po(resources, translations=None, resfilter=None, warnfunc=dummy_warn):
"""Return ``resources`` as a Babel .po ``Catalog`` instance.
If given, ``translations`` will be used for the translated values.
In this case, the returned value is a 2-tuple (catalog, unmatched),
with the latter being a list of Android string resource names that
are in the translated file, but not in the original.
Both ``resources`` and ``translations`` must be ``ResourceTree``
objects, as returned by ``read_xml()``.
From the application perspective, it will call this function with
a ``translations`` object when initializing a new .po file based on
an existing resource file (the 'init' command). For 'export', this
function is called without translations. It will thus generate what
is essentially a POT file (an empty .po file), and this will be
merged into the existing .po catalogs, as per how gettext usually
"""
assert not translations or translations.language
catalog = Catalog()
if translations is not None:
catalog.locale = translations.language.locale
# We cannot let Babel determine the plural expr for the locale by
# itself. It will use a custom list of plural expressions rather
# than generate them based on CLDR.
# See http://babel.edgewall.org/ticket/290.
set_catalog_plural_forms(catalog, translations.language)
for name, org_value in resources.items():
if resfilter and resfilter(name):
continue
trans_value = None
if translations:
trans_value = translations.pop(name, trans_value)
if isinstance(org_value, StringArray):
# a string-array, write as "name:index"
if len(org_value) == 0:
warnfunc("Warning: string-array '%s' is empty" % name,
'warning')
continue
if not isinstance(trans_value, StringArray):
if trans_value:
warnfunc(('""%s" is a string-array in the reference '
'file, but not in the translation.') % name,
'warning')
trans_value = StringArray()
for index, item in enumerate(org_value):
item_trans = trans_value[index].text if index < len(
trans_value) else ''
# If the string has formatting markers, indicate it in
# the gettext output
flags = []
if item.formatted:
flags.append('c-format')
ctx = "%s:%d" % (name, index)
catalog.add(item.text,
item_trans,
auto_comments=item.comments,
flags=flags,
context=ctx)
elif isinstance(org_value, Plurals):
# a plurals, convert to a gettext plurals
if len(org_value) == 0:
warnfunc("Warning: plurals '%s' is empty" % name, 'warning')
continue
if not isinstance(trans_value, Plurals):
if trans_value:
warnfunc(('""%s" is a plurals in the reference '
'file, but not in the translation.') % name,
'warning')
trans_value = Plurals()
# Taking the Translation objects for each quantity in ``org_value``,
# we build a list of strings, which is how plurals are represented
# in Babel.
#
# Since gettext only allows comments/flags on the whole
# thing at once, we merge the comments/flags of all individual
# plural strings into one.
formatted = False
comments = []
for _, translation in list(org_value.items()):
if translation.formatted:
formatted = True
comments.extend(translation.comments)
# For the message id, choose any two plural forms, but prefer
# "one" and "other", assuming an English master resource.
temp = org_value.copy()
singular =\
temp.pop('one') if 'one' in temp else\
temp.pop('other') if 'other' in temp else\
temp.pop(list(temp.keys())[0])
plural =\
temp.pop('other') if 'other' in temp else\
temp[list(temp.keys())[0]] if temp else\
singular
msgid = (singular.text, plural.text)
del temp, singular, plural
# We pick the quantities supported by the language (the rest
# would be ignored by Android as well).
msgstr = ''
if trans_value:
allowed_keywords = translations.language.plural_keywords
msgstr = ['' for i in range(len(allowed_keywords))]
for quantity, translation in list(trans_value.items()):
try:
index = translations.language.plural_keywords.index(
quantity)
except ValueError:
warnfunc(
('"plurals "%s" uses quantity "%s", which '
'is not supported for this language. See '
'the README for an explanation. The '
'quantity has been ignored') % (name, quantity),
'warning')
else:
msgstr[index] = translation.text
flags = []
if formatted:
flags.append('c-format')
catalog.add(msgid,
tuple(msgstr),
flags=flags,
auto_comments=comments,
context=name)
else:
# a normal string
# If the string has formatting markers, indicate it in
# the gettext output
# TODO DRY this.
flags = []
if org_value.formatted:
flags.append('c-format')
catalog.add(org_value.text,
trans_value.text if trans_value else '',
flags=flags,
auto_comments=org_value.comments,
context=name)
if translations is not None:
# At this point, trans_strings only contains those for which
# no original existed.
return catalog, list(translations.keys())
else:
return catalog
def do_extract_messages(target=('t', ''), domain=('d', 'messages'),
i18n_dir=('i', ''), all=('a', False)):
"""
Extract messages and create pot file.
"""
if not domain in ('messages', 'jsmessages'):
print_status('invalid domain.')
sys.exit(1)
if not target and not all:
print_status('Please specify target.')
sys.exit(1)
elif target == 'kay':
print_status('Extracting core strings')
root = kay.KAY_DIR
elif all:
targets = get_user_apps()
for target in targets:
do_extract_messages(target=target, domain=domain, i18n_dir=None,
all=False)
sys.exit(0)
else:
root = path.abspath(target)
if not path.isdir(root):
print_status('source folder missing')
sys.exit(1)
print_status('Extracting from %s' % root)
if domain == 'messages':
methods = METHODS
else:
methods = JSMETHODS
catalog = Catalog(msgid_bugs_address=BUGS_ADDRESS,
copyright_holder=COPYRIGHT, charset='utf-8')
def callback(filename, method, options):
if method != 'ignore':
print_status(strip_path(filename, root))
option = {}
option['extensions'] = ','.join(settings.JINJA2_EXTENSIONS)
option.update(settings.JINJA2_ENVIRONMENT_KWARGS)
options = {
'**/templates/**.*': option,
'**.html': option,
}
extracted = extract_from_dir(root, methods, options, KEYWORDS,
COMMENT_TAGS, callback=callback,
strip_comment_tags=True)
for filename, lineno, message, comments in extracted:
catalog.add(message, None, [(strip_path(filename, root), lineno)],
auto_comments=comments)
if not i18n_dir:
i18n_dir = path.join(root, 'i18n')
if not path.isdir(i18n_dir):
makedirs(i18n_dir)
f = file(path.join(i18n_dir, domain+'.pot'), 'w')
try:
write_po(f, catalog, width=79)
finally:
f.close()
print_status('All done.')
for loc in sorted(args.locale):
try:
locale = Locale.parse(loc)
except UnknownLocaleError as e:
logging.error(e.args[0])
continue
catalog = Catalog(locale,
domain,
header,
fuzzy=None,
project='gettext-CLDR-translations',
version='1.0',
copyright_holder='Tomasz Słodkowicz')
for c in sorted(locale.currencies):
catalog.add(C.currencies[c], locale.currencies[c], auto_comments=[c])
subdir = catalog.locale_identifier
if args.po:
makedirs('po/%s' % subdir, exist_ok=True)
with open('po/%s/%s.po' % (subdir, domain), 'wb') as f:
pofile.write_po(f, catalog)
if args.mo:
makedirs('mo/%s' % subdir, exist_ok=True)
with open('mo/%s/%s.mo' % (subdir, domain), 'wb') as f:
mofile.write_mo(f, catalog)
if args.stats:
print('%3i%% %s' %
(len(catalog._messages) * 100 / max_cur, locale.english_name))
t += len(catalog._messages)
def xml2po(resources, translations=None, filter=None, warnfunc=dummy_warn):
"""Return ``resources`` as a Babel .po ``Catalog`` instance.
If given, ``translations`` will be used for the translated values.
In this case, the returned value is a 2-tuple (catalog, unmatched),
with the latter being a list of Android string resource names that
are in the translated file, but not in the original.
Both ``resources`` and ``translations`` must be ``ResourceTree``
objects, as returned by ``read_xml()``.
From the application perspective, it will call this function with
a ``translations`` object when initializing a new .po file based on
an existing resource file (the 'init' command). For 'export', this
function is called without translations. It will thus generate what
is essentially a POT file (an empty .po file), and this will be
merged into the existing .po catalogs, as per how gettext usually
"""
assert not translations or translations.language
catalog = Catalog()
if translations is not None:
catalog.locale = translations.language.locale
# We cannot let Babel determine the plural expr for the locale by
# itself. It will use a custom list of plural expressions rather
# than generate them based on CLDR.
# See http://babel.edgewall.org/ticket/290.
set_catalog_plural_forms(catalog, translations.language)
for name, org_value in resources.iteritems():
if filter and filter(name):
continue
trans_value = None
if translations:
trans_value = translations.pop(name, trans_value)
if isinstance(org_value, StringArray):
# a string-array, write as "name:index"
if len(org_value) == 0:
warnfunc("Warning: string-array '%s' is empty" % name, "warning")
continue
if not isinstance(trans_value, StringArray):
if trans_value:
warnfunc(
('""%s" is a string-array in the reference ' "file, but not in the translation.") % name,
"warning",
)
trans_value = StringArray()
for index, item in enumerate(org_value):
item_trans = trans_value[index].text if index < len(trans_value) else u""
# If the string has formatting markers, indicate it in
# the gettext output
flags = []
if item.formatted:
flags.append("c-format")
ctx = "%s:%d" % (name, index)
catalog.add(item.text, item_trans, auto_comments=item.comments, flags=flags, context=ctx)
elif isinstance(org_value, Plurals):
# a plurals, convert to a gettext plurals
if len(org_value) == 0:
warnfunc("Warning: plurals '%s' is empty" % name, "warning")
continue
if not isinstance(trans_value, Plurals):
if trans_value:
warnfunc(
('""%s" is a plurals in the reference ' "file, but not in the translation.") % name, "warning"
)
trans_value = Plurals()
# Taking the Translation objects for each quantity in ``org_value``,
# we build a list of strings, which is how plurals are represented
# in Babel.
#
# Since gettext only allows comments/flags on the whole
# thing at once, we merge the comments/flags of all individual
# plural strings into one.
formatted = False
comments = []
for _, translation in org_value.items():
if translation.formatted:
formatted = True
comments.extend(translation.comments)
# For the message id, choose any two plural forms, but prefer
# "one" and "other", assuming an English master resource.
temp = org_value.copy()
singular = (
temp.pop("one") if "one" in temp else temp.pop("other") if "other" in temp else temp.pop(temp.keys()[0])
)
plural = temp.pop("other") if "other" in temp else temp[temp.keys()[0]] if temp else singular
msgid = (singular.text, plural.text)
del temp, singular, plural
# We pick the quantities supported by the language (the rest
# would be ignored by Android as well).
msgstr = ""
if trans_value:
allowed_keywords = translations.language.plural_keywords
msgstr = ["" for i in range(len(allowed_keywords))]
for quantity, translation in trans_value.items():
try:
index = translations.language.plural_keywords.index(quantity)
except ValueError:
warnfunc(
(
'"plurals "%s" uses quantity "%s", which '
"is not supported for this language. See "
"the README for an explanation. The "
"quantity has been ignored"
)
% (name, quantity),
"warning",
)
else:
msgstr[index] = translation.text
flags = []
if formatted:
flags.append("c-format")
catalog.add(msgid, tuple(msgstr), flags=flags, auto_comments=comments, context=name)
else:
# a normal string
# If the string has formatting markers, indicate it in
# the gettext output
# TODO DRY this.
flags = []
if org_value.formatted:
flags.append("c-format")
catalog.add(
org_value.text,
trans_value.text if trans_value else u"",
flags=flags,
auto_comments=org_value.comments,
context=name,
)
if translations is not None:
# At this point, trans_strings only contains those for which
# no original existed.
return catalog, translations.keys()
else:
return catalog
def test_more_plural_forms(self):
catalog2 = Catalog(locale="ru_RU")
catalog2.add(("Fuzz", "Fuzzes"), ("", "", ""))
buf = StringIO()
mofile.write_mo(buf, catalog2)
from datetime import datetime, timedelta
from io import BytesIO
import babel.dates
from babel.messages import Catalog
from babel.messages.mofile import write_mo
from cms.locale import Translation, DEFAULT_TRANSLATION, filter_language_codes, \
choose_language_code
UTC = babel.dates.UTC
ROME = babel.dates.get_timezone("Europe/Rome")
FRENCH_CATALOG = Catalog(locale="fr")
FRENCH_CATALOG.add("%s KiB", "%s Kio")
FRENCH_CATALOG.add("%s MiB", "%s Mio")
FRENCH_CATALOG.add("%s GiB", "%s Gio")
FRENCH_CATALOG.add("%s TiB", "%s Tio")
FRENCH_CATALOG_BUF = BytesIO()
write_mo(FRENCH_CATALOG_BUF, FRENCH_CATALOG)
FRENCH_CATALOG_BUF.seek(0)
ENGLISH = DEFAULT_TRANSLATION
BRITISH_ENGLISH = Translation("en_GB")
FRENCH = Translation("fr", mofile=FRENCH_CATALOG_BUF)
HINDI = Translation("hi")
ITALIAN = Translation("it")
NORWEGIAN = Translation("no")
CHINESE = Translation("zh_CN")
import unittest
from datetime import datetime, timedelta
from io import BytesIO
import babel.dates
from babel.messages import Catalog
from babel.messages.mofile import write_mo
from cms.locale import Translation, DEFAULT_TRANSLATION, filter_language_codes, \
choose_language_code
UTC = babel.dates.UTC
ROME = babel.dates.get_timezone("Europe/Rome")
FRENCH_CATALOG = Catalog(locale="fr")
FRENCH_CATALOG.add("%s KiB", "%s Kio")
FRENCH_CATALOG.add("%s MiB", "%s Mio")
FRENCH_CATALOG.add("%s GiB", "%s Gio")
FRENCH_CATALOG.add("%s TiB", "%s Tio")
FRENCH_CATALOG_BUF = BytesIO()
write_mo(FRENCH_CATALOG_BUF, FRENCH_CATALOG)
FRENCH_CATALOG_BUF.seek(0)
ENGLISH = DEFAULT_TRANSLATION
BRITISH_ENGLISH = Translation("en_GB")
FRENCH = Translation("fr", mofile=FRENCH_CATALOG_BUF)
HINDI = Translation("hi")
ITALIAN = Translation("it")
NORWEGIAN = Translation("no")
CHINESE = Translation("zh_CN")
def test_more_plural_forms(self):
catalog2 = Catalog(locale='ru_RU')
catalog2.add(('Fuzz', 'Fuzzes'), ('', '', ''))
buf = StringIO()
mofile.write_mo(buf, catalog2)
missing_key_msg = validation.MESSAGES['MISSING_KEY']
not_a_number_msg = validation.MESSAGES['NOT_A_NUMBER']
# Fake translations for testing
TEST_MESSAGES = {
"InVEST Carbon Model": "ιиνєѕт ςαявσи мσ∂єℓ",
"Available models:": "αναιℓαвℓє мσ∂єℓѕ:",
"Carbon Storage and Sequestration": "ςαявσи ѕтσяαgє αи∂ ѕєףυєѕтяαтισи",
"current LULC": "ςυяяєит ℓυℓς",
missing_key_msg: "кєу ιѕ мιѕѕιиg fяσм тнє αяgѕ ∂ιςт",
not_a_number_msg: 'ναℓυє "{value}" ςσυℓ∂ иσт вє ιитєяρяєтє∂ αѕ α иυмвєя'
}
TEST_CATALOG = Catalog(locale=TEST_LANG)
for key, value in TEST_MESSAGES.items():
TEST_CATALOG.add(key, value)
def reset_locale():
"""Reset affected parts of the global context."""
set_locale('en')
# "unimport" the modules being translated
# NOTE: it would be better to run each test in a new process,
# but that's difficult on windows: https://stackoverflow.com/a/48310939
importlib.reload(natcap.invest)
importlib.reload(validation)
importlib.reload(cli)
importlib.reload(carbon)
importlib.reload(ui_server)
