def test_plural(self):
entry = POEntry()
entry.msgid = "A lovely day for a cup of tea."
entry.msgid_plural = "A lovely day for some cups of tea."
expected_s = u"À løvélý däý før ä çüp øf téä. Ⱡ'σяєм #"
expected_p = u"À løvélý däý før sømé çüps øf téä. Ⱡ'σяєм ιρ#"
self.converter.convert_msg(entry)
result = entry.msgstr_plural
self.assertUnicodeEquals(result['0'], expected_s)
self.assertUnicodeEquals(result['1'], expected_p)
def test_plural(self):
entry = POEntry()
entry.msgid = 'A lovely day for a cup of tea.'
entry.msgid_plural = 'A lovely day for some cups of tea.'
expected_s = u'\xc0 l\xf6v\xe9ly d\xe4y f\xf6r \xe4 \xe7\xfcp \xf6f t\xe9\xe4. Lorem i#'
expected_p = u'\xc0 l\xf6v\xe9ly d\xe4y f\xf6r s\xf6m\xe9 \xe7\xfcps \xf6f t\xe9\xe4. Lorem ip#'
self.converter.convert_msg(entry)
result = entry.msgstr_plural
self.assertEquals(result['0'], expected_s)
self.assertEquals(result['1'], expected_p)
def test_plural(self):
entry = POEntry()
entry.msgid = "A lovely day for a cup of tea."
entry.msgid_plural = "A lovely day for some cups of tea."
expected_s = u"À lövélý däý för ä çüp öf téä. Ⱡ'σяєм ιρѕυм ∂σłσя ѕιт αмєт, ¢σηѕє¢т#"
expected_p = u"À lövélý däý för sömé çüps öf téä. Ⱡ'σяєм ιρѕυм ∂σłσя ѕιт αмєт, ¢σηѕє¢тєт#"
self.converter.convert_msg(entry)
result = entry.msgstr_plural
self.assertUnicodeEquals(result['0'], expected_s)
self.assertUnicodeEquals(result['1'], expected_p)
def test_success(self):
self.assertTrue(testString(POEntry()))
def test_no_comment(self):
self.assertRaises(AssertionError, test_comment, POEntry(msgid="c"))
def test_fail(self):
entry = POEntry(msgid="Source", msgstr="Translation", obsolete=True)
status = Status()
status.step(entry)
self.assertFalse(obsolete_validator(status))
def test_singular(self):
entry = POEntry()
entry.msgid = "A lovely day for a cup of tea."
expected = u"À løvélý däý før ä çüp øf téä. Ⱡ'σяєм #"
self.converter.convert_msg(entry)
self.assertUnicodeEquals(entry.msgstr, expected)
def entry(msgid, msgstr, comment=""):
po.append(POEntry(
msgid=msgid,
msgstr=msgstr,
comment=comment,
))
def test_singular(self):
entry = POEntry()
entry.msgid = "A lovely day for a cup of tea."
expected = u"À lövélý däý för ä çüp öf téä. Ⱡ'σяєм ιρѕυм ∂σłσя ѕιт αмєт, ¢σηѕє¢т#"
self.converter.convert_msg(entry)
self.assertUnicodeEquals(entry.msgstr, expected)
def extract_countries_po():
input_path = get_shape_path("ne_50m_admin_0_countries")
for locale in os.listdir(LOCALE_DIR):
locale_dir = path.join(LOCALE_DIR, locale)
locale_out_dir = path.join(LOCALE_OUT_DIR, locale)
if os.path.isdir(locale_dir):
with fiona.open(input_path) as source:
po = POFile(encoding='UTF-8')
po.metadata = {"Content-Type": "text/plain; charset=utf-8"}
output_path = path.join(locale_out_dir, "countries.po")
if not path.exists(locale_out_dir):
os.makedirs(locale_out_dir)
print "Generating {}/countries.po".format(locale)
for feat in source:
props = lower_dict_keys(feat["properties"])
name_key = "_".join(("name", get_locale_language(locale)))
name_alt_key = "_".join(("name", convert_locale_ident(locale)))
name_fallback = "name"
country_name = props.get("name")
formal_country_name = props.get("formal_en", country_name)
if props.get(name_key) is not None:
translated_name = props.get(name_key)
elif props.get(name_alt_key) is not None:
translated_name = props.get(name_alt_key)
elif props.get(name_fallback) is not None:
translated_name = props.get(name_fallback)
print c.orange(u" Missing translation for {}".format(translated_name))
else:
raise ValueError(
"Cannot find the translation for {}. Probe keys: {}"
.format(locale, (name_key, name_alt_key))
)
entry = POEntry(
msgid=country_name,
msgstr=translated_name
)
po.append(entry)
# add additional record for the formal country name.
if country_name != formal_country_name and formal_country_name is not None:
entry = POEntry(
msgid=formal_country_name,
msgstr=translated_name
)
po.append(entry)
# exception for the US
if props.get("iso_a3") == "USA":
entry = POEntry(
msgid="USA",
msgstr=translated_name
)
po.append(entry)
# exception for the UK
if props.get("iso_a3") == "GBR":
entry = POEntry(
msgid="UK",
msgstr=translated_name
)
po.append(entry)
po.save(output_path)
print c.green("Extracted {} countries for {} to {}".format(len(po), locale, output_path))
def test_singular(self):
entry = POEntry()
entry.msgid = 'A lovely day for a cup of tea.'
expected = u'\xc0 l\xf6v\xe9ly d\xe4y f\xf6r \xe4 \xe7\xfcp \xf6f t\xe9\xe4. Lorem i#'
self.converter.convert_msg(entry)
self.assertEquals(entry.msgstr, expected)
def test_first_entry(self):
entry = POEntry(msgid="Source", msgstr="Translation")
status = Status()
status.step(entry)
self.assertTrue(sort_validator(status))
def test_pass(self):
entry = POEntry(msgid="Source", msgstr="Translation")
status = Status()
status.step(entry)
self.assertTrue(no_location_validator(status))
def test_fail_obsolete(self):
# Obsolete translations are considered untranslated
entry = POEntry(msgid="Source", msgstr="Translation", obsolete=True)
status = Status()
status.step(entry)
self.assertFalse(untranslated_validator(status))
def test_fail_fuzzy(self):
# Fuzzy translations are considered untranslated
entry = POEntry(msgid="Source", msgstr="Translation", flags=['fuzzy'])
status = Status()
status.step(entry)
self.assertFalse(untranslated_validator(status))
def test_fail_missing(self):
entry = POEntry(msgid="Source", msgstr="")
status = Status()
status.step(entry)
self.assertFalse(untranslated_validator(status))
def test_failure(self):
self.assertFalse(testString(POEntry()))
def test_singular(self):
entry = POEntry()
entry.msgid = "A lovely day for a cup of tea."
expected = u"À lövélý däý för ä çüp öf téä. Ⱡ'σяєм ιρѕυм ∂σłσя ѕιт αмєт, ¢σηѕє¢т#"
self.converter.convert_msg(entry)
self.assertUnicodeEquals(entry.msgstr, expected)
def command_main_po(from_language, to_language, input, output=None, po=None, pot=None, pdf=False, undetected_strings=[], fake=None):
extension=input.split(".")[1:][0]
if output is None:
output=f"{input}.{to_language}.{extension}"
if po is None:
po=f"{input}.{to_language}.po"
if pot is None:
pot=f"{input}.{to_language}.pot"
logfile=f"{output}.log"
log=open(logfile, "w")
s=_(f"Translating '{input}' from '{from_language }' to '{to_language}'")
print(s)
log.write(s+"\n")
s=_(f" - Output: {output}")
print(s)
log.write(s+"\n")
s=_(f" - File catalog pot: {pot}")
print(s)
log.write(s+"\n")
s=_(f" - File catalog po: {po}")
print(s)
log.write(s+"\n")
s=_(f" - Translation log: {logfile}")
print(s)
log.write(s+"\n")
original_xlf="original.xlf"
if path.exists(original_xlf) is True:
remove(original_xlf)
doc=ODT(input)
# Creating a list of ordered document strings
run_check(["odf2xliff", input, original_xlf])
##Leemos sources
mytree = ET.parse(original_xlf)
myroot = mytree.getroot()
file_=myroot[0]
body=file_[0]
sources=set()
for e in body:
if e[0].text is not None:#<source>
s=innercontent(e[0])
arr=removeTags(s)
for t in arr:
sources.add(t)
for s in undetected_strings:
sources.add(s)
sources=list(sources)
if path.exists(original_xlf) is True:
remove(original_xlf)
# Creating pot file
file_pot = POFile()
file_pot.metadata = {
'Project-Id-Version': '1.0',
'Report-Msgid-Bugs-To': '*****@*****.**',
'POT-Creation-Date': '2007-10-18 14:00+0100',
'PO-Revision-Date': '2007-10-18 14:00+0100',
'Last-Translator': 'you <*****@*****.**>',
'Language-Team': 'English <*****@*****.**>',
'MIME-Version': '1.0',
'Content-Type': 'text/plain; charset=utf-8',
'Content-Transfer-Encoding': '8bit',
}
for i, source in enumerate(sources):
entry = POEntry(
msgid=source,
msgstr='',
occurrences=[('string', str(i)),]
)
file_pot.append(entry)
file_pot.save(pot)
#Merging pot with out po file
if path.exists(po)==False:
run_check(["msginit", "-i", pot, "-o", po])
run_check(["msgmerge","-N", "--no-wrap","-U", po, pot])
# Creates a dictionary of translations
dict_po={}
file_po = pofile(po)
for i, entry in enumerate(file_po):
if fake is True:
dict_po[entry.msgid]=f"{{{entry.msgid}}}"
else:
if entry.msgstr == "":
dict_po[entry.msgid]=entry.msgid
else:
dict_po[entry.msgid]=entry.msgstr
#Converts sources to entries (list of tuples)
entries=[]
for source in sources:
entries.append((source, dict_po[source] ))
entries=sorted(entries, key=lambda item: len(item[0]), reverse=True)
#Creating our translated output
log.write ("\n\n==== TRANSLATION LOGS ====\n")
warns=""
for i, (find, replace) in enumerate(entries):
number=doc.findall_and_replace(find, replace)
rs=replaced_entries_before(find, i, entries)
s=f"""
* Entry {i}
Original: {find}
Translation set {number} times: {replace}
"""
log.write(s)
if len(rs)>0:
warns=warns + s +" WARNING: This replacement could overwrite before replacements. Perhaps you'll need to overwrite your result later with unogenerator.\n"
for s in rs:
warns =warns + f" - '{s[0]}' ==> '{s[1]}'\n"
if warns!="":
log.write ("\n\n==== WARNINGS ====\n")
log.write(warns)
doc.save(output)
if pdf is True:
doc.export_pdf(output+".pdf")
doc.close()
print(f"{len(sources)} messages found. {len(file_po.translated_entries())} translated. {len(file_po.untranslated_entries())} untranslated.")
log.close()
def translate_relay_locations(place_translator, countries, locale):
po = POFile(encoding='UTF-8')
po.metadata = {"Content-Type": "text/plain; charset=utf-8"}
locale_out_dir = path.join(LOCALE_OUT_DIR, locale)
output_path = path.join(locale_out_dir, "relay-locations.po")
hits = 0
misses = 0
if not path.exists(locale_out_dir):
os.makedirs(locale_out_dir)
for country in countries:
country_name = country.get("name")
country_code = country.get("code")
cities = country.get("cities")
if cities is None:
print c.orange(
u"Skip {} ({}) because no cities were found.".format(
country_name, country_code))
continue
for city in cities:
city_name = city.get("name")
city_code = city.get("code")
if city_name is None:
raise ValueError("Missing the name field in city record.")
# Make sure to append the US state back to the translated name of the city
if country_code == "us":
split = city_name.rsplit(",", 2)
translated_name = place_translator.translate(
locale, split[0].strip())
if translated_name is not None and len(split) > 1:
translated_name = u"{}, {}".format(translated_name,
split[1].strip())
else:
translated_name = place_translator.translate(locale, city_name)
# Default to empty string if no translation was found
found_translation = translated_name is not None
if found_translation:
hits += 1
else:
translated_name = ""
misses += 1
log_message = u" {} ({}) -> \"{}\"".format(
city_name, city_code, translated_name).encode('utf-8')
if found_translation:
print c.green(log_message)
else:
print c.orange(log_message)
entry = POEntry(msgid=city_name,
msgstr=translated_name,
comment=u"{} {}".format(
country.get("code").upper(),
city.get("code").upper()))
po.append(entry)
po.save(output_path)
return (hits, misses)
def command_generate_po(from_language, to_language, input, output_directory, translate, undetected_strings=[]):
def same_entries_to_ocurrences(l):
l= sorted(l, key=lambda x: (x[1], x[2], x[3]))
r=[]
for filename, type, number, position, text in l:
r.append((filename, f"{type}#{number}#{position}"))
return r
##########################
makedirs(output_directory, exist_ok=True)
makedirs(f"{output_directory}/{to_language}", exist_ok=True)
pot=f"{output_directory}/catalogue.pot"
po=f"{output_directory}/{to_language}/{to_language}.po"
entries=[]#List of ("type", numero, posicion) type=Paragraph, numero=numero parrafo y posición orden dentro del parrafo
set_strings=set()
# Creating pot file
print(_("Extracting strings from:"))
for filename in input:
print(_(f" - {filename}"))
doc=ODT(filename)
#Extract strings from paragraphs
enumeration = doc.cursor.Text.createEnumeration()
for i, par in enumerate(enumeration):
if par.supportsService("com.sun.star.text.Paragraph") :
for position, element in enumerate(par.createEnumeration()):
text_=element.getString()
if text_ !="" and text_!=" " and text_!=" ":
entries.append((filename,"Paragraph", i, position, text_))
set_strings.add(text_)
doc.close()
#Extract strings from headers
# ' Turn headers on and then make them shared!
#oPstyle.HeaderOn = True
#oPstyle.HeaderShared = True
#' The is also a RightText and a LeftText
#oHeader = oPstyle.RightPageHeaderContent
#oText = oHeader.CenterText
#' You may now set the text object to be anything you desire
#' Use setSTring() from the text object to set simple text.
#' Use a cursor to insert a field (such as the current sheet name).
#' First, clear any existing text!
#oText.setString("")
#oCursor = oText.createTextCursor()
#oText.insertString(oCursor, "Sheet: ", False)
#' This will have the sheet name of the current sheet!
#sService = "com.sun.star.text.TextField.SheetName"
#oField = oDoc.createInstance(sService)
#oText.insertTextContent(oCursor, oField, False)
file_pot = POFile()
file_pot.metadata = {
'Project-Id-Version': '1.0',
'Report-Msgid-Bugs-To': '*****@*****.**',
'POT-Creation-Date': '2007-10-18 14:00+0100',
'PO-Revision-Date': '2007-10-18 14:00+0100',
'Last-Translator': 'you <*****@*****.**>',
'Language-Team': 'English <*****@*****.**>',
'MIME-Version': '1.0',
'Content-Type': 'text/plain; charset=utf-8',
'Content-Transfer-Encoding': '8bit',
}
for s in set_strings:
same_entries=[] #Join seame text entries
for filename, type, number, position, string_ in entries:
if string_==s:
same_entries.append((filename, type, number, position, string_))
entry = POEntry(
msgid=s,
msgstr='',
occurrences=same_entries_to_ocurrences(same_entries)
)
file_pot.append(entry)
file_pot.save(pot)
#Merging pot with out po file
if path.exists(po)==False:
run_check(["msginit", "-i", pot, "-o", po])
run_check(["msgmerge","-N", "--no-wrap","-U", po, pot])
print(f"{len(set_strings)} different strings detected")
if translate is True:
print(_("Translating files to:"))
for filename in input:
doc=ODT(filename)
output=f"{output_directory}/{to_language}/{path.basename(filename)}"
print(_(f" - {output}"))
doc.save(output)
doc.close()
def handle(self, input_directory, **options):
if options["versions"]:
versions_to_include = options["versions"].split(",")
else:
versions_to_include = None
if options["languages"]:
languages_to_include = set(["en"]) | set(options["languages"].split(","))
else:
languages_to_include = None
self.unwrapped = options["unwrapped"]
licenses_created = 0
legalcodes_created = 0
legalcodes_to_import = []
# Get list of html filenames for CC0 and any BY license (any version).
# We'll filter out the filenames for unwanted versions later.
html_filenames = sorted(
[
f
for f in os.listdir(input_directory)
if (f.startswith("by") or f.startswith("zero_1.0"))
and f.endswith(".html")
]
)
for filename in html_filenames:
# print(filename)
metadata = parse_legalcode_filename(filename)
basename = os.path.splitext(filename)[0]
fullpath = os.path.join(input_directory, filename)
license_code = metadata["license_code"]
version = metadata["version"]
jurisdiction_code = metadata["jurisdiction_code"]
cc_language_code = metadata[
"cc_language_code"
] or get_default_language_for_jurisdiction(jurisdiction_code)
# Make sure this is a valid language code (one we know about)
django_language_code = cc_to_django_language_code(cc_language_code)
if django_language_code not in settings.LANG_INFO:
raise ValueError(f"Invalid language_code={cc_language_code}")
# Just CC0, BY 3.0, & 4.0, and apply any command line options
include = (
(
(license_code in BY_LICENSE_CODES and version in {"3.0", "4.0"})
or license_code in CC0_LICENSE_CODES
)
and (versions_to_include is None or version in versions_to_include)
and (
languages_to_include is None
or cc_language_code in languages_to_include
)
)
if not include:
continue
about_url = metadata["about_url"]
# These are valid for BY only
license_code_parts = license_code.split("-")
if "by" in license_code_parts:
permits_derivative_works = "nd" not in license_code_parts
permits_reproduction = "nd" not in license_code_parts
permits_distribution = "nd" not in license_code_parts
permits_sharing = "nd" not in license_code_parts
requires_share_alike = "sa" in license_code_parts
requires_notice = True
requires_attribution = True
requires_source_code = False # GPL, LGPL only, I think
prohibits_commercial_use = "nc" in license_code_parts
prohibits_high_income_nation_use = False # Not any BY 4.0 license
elif license_code == "CC0":
# permits anything, requires nothing, prohibits nothing
permits_derivative_works = True
permits_reproduction = True
permits_distribution = True
permits_sharing = True
requires_share_alike = False
requires_notice = False
requires_attribution = False
requires_source_code = False
prohibits_commercial_use = False
prohibits_high_income_nation_use = False
else:
raise NotImplementedError(basename)
# Find or create a License object
license, created = License.objects.get_or_create(
about=about_url,
defaults=dict(
license_code=license_code,
version=version,
jurisdiction_code=jurisdiction_code,
permits_derivative_works=permits_derivative_works,
permits_reproduction=permits_reproduction,
permits_distribution=permits_distribution,
permits_sharing=permits_sharing,
requires_share_alike=requires_share_alike,
requires_notice=requires_notice,
requires_attribution=requires_attribution,
requires_source_code=requires_source_code,
prohibits_commercial_use=prohibits_commercial_use,
prohibits_high_income_nation_use=prohibits_high_income_nation_use,
),
)
if created:
licenses_created += 1
# Find or create a LegalCode object
legalcode, created = LegalCode.objects.get_or_create(
license=license,
language_code=cc_language_code,
defaults=dict(
html_file=fullpath,
),
)
if created:
legalcodes_created += 1
legalcodes_to_import.append(legalcode)
# print(
# f"Created {licenses_created} licenses and {legalcodes_created} translation objects"
# )
# NOW parse the HTML and output message files
legalcodes_to_import = LegalCode.objects.filter(
pk__in=[lc.pk for lc in legalcodes_to_import]
)
# What are the language codes we have HTML files for?
cc_language_codes = sorted(set(lc.language_code for lc in legalcodes_to_import))
english_by_license_code_version = {}
# We have to do English first. Django gets confused if you try to load
# another language and it can't find English, I guess it's looking for
# something to fall back to.
cc_language_codes.remove(
"en"
) # If english isn't in this list, something is wrong
for cc_language_code in ["en"] + cc_language_codes:
for legalcode in legalcodes_to_import.filter(
language_code=cc_language_code,
).order_by(
"-license__version",
"license__license_code",
"license__jurisdiction_code",
):
license = legalcode.license
license_code = license.license_code
version = license.version
# print(
# f"Importing {legalcode.html_file} {license_code} lang={cc_language_code}"
# )
with open(legalcode.html_file, "r", encoding="utf-8") as f:
content = f.read()
if version == "4.0":
messages_text = self.import_by_40_license_html(
content=content,
legalcode=legalcode,
)
elif version == "3.0":
if license.jurisdiction_code:
# Ported license: we just save the HTML for now
legalcode.html = self.import_by_30_ported_license_html(
content=content,
legalcode=legalcode,
)
legalcode.save()
continue
else:
# Unported license: we parse out the messages like 4.0
messages_text = self.import_by_30_unported_license_html(
content=content,
legalcode=legalcode,
)
elif license_code == "CC0":
messages_text = self.import_cc0_license_html(
content=content,
legalcode=legalcode,
)
else:
raise NotImplementedError(
f"Have not implemented parsing for {license_code} {version} licenses."
)
if version != "3.0":
# 3.0 doesn't have any translation files - might be the same for other versions
key = f"{license_code}|{version}"
if cc_language_code == "en":
english_by_license_code_version[key] = messages_text
english_messages = english_by_license_code_version[key]
pofile = POFile()
# The syntax used to wrap messages in a .po file is difficult if you ever
# want to copy/paste the messages, so if --unwrapped was passed, set a
# wrap width that will essentially disable wrapping.
if self.unwrapped:
pofile.wrapwidth = 999999
pofile.metadata = {
"Project-Id-Version": f"{license_code}-{version}",
# 'Report-Msgid-Bugs-To': '*****@*****.**',
# 'POT-Creation-Date': '2007-10-18 14:00+0100',
# 'PO-Revision-Date': '2007-10-18 14:00+0100',
# 'Last-Translator': 'you <*****@*****.**>',
# 'Language-Team': 'English <*****@*****.**>',
"Language": cc_language_code,
"MIME-Version": "1.0",
"Content-Type": "text/plain; charset=utf-8",
"Content-Transfer-Encoding": "8bit",
}
# Use the English message text as the message key
for internal_key, translation in messages_text.items():
if cc_language_code == "en":
message_key = translation.strip()
message_value = ""
else:
# WORKAROUND - by-nc-nd 4.0 NL has an extra item under s3a.
# https://github.com/creativecommons/creativecommons.org/pull/1160
if (
internal_key == "s3a4_if_you_share_adapted_material"
and internal_key not in english_messages
):
message_key = (
"If You Share Adapted Material You produce, the Adapter's "
"License You apply must not prevent recipients of the Adapted "
"Material from complying with this Public License."
)
else:
message_key = english_messages[internal_key]
message_value = translation
pofile.append(
POEntry(
msgid=clean_string(message_key),
msgstr=clean_string(message_value),
)
)
po_filename = legalcode.translation_filename()
dir = os.path.dirname(po_filename)
if not os.path.isdir(dir):
os.makedirs(dir)
# Save mofile ourself. We could call 'compilemessages' but it wants to
# compile everything, which is both overkill and can fail if the venv
# or project source is not writable. We know this dir is writable, so
# just save this pofile and mofile ourselves.
save_pofile_as_pofile_and_mofile(pofile, po_filename)
def test_singular(self):
entry = POEntry()
entry.msgid = 'A lovely day for a cup of tea.'
expected = u'\xc0 l\xf6v\xe9ly d\xe4y f\xf6r \xe4 \xe7\xfcp \xf6f t\xe9\xe4. Lorem i#'
self.converter.convert_msg(entry)
self.assertEquals(entry.msgstr, expected)
def test_singular(self):
entry = POEntry()
entry.msgid = "A lovely day for a cup of tea."
expected = u"À løvélý däý før ä çüp øf téä. Ⱡ'σяєм #"
self.converter.convert_msg(entry)
self.assertUnicodeEquals(entry.msgstr, expected)
def translate_single_relay_locations(country_translator, city_translator,
countries, locale):
"""
A helper function to generate the relay-locations.po for the given locale.
The `countries` argument is an array value that's contained within the "countries" key of the
relay location list.
"""
po = POFile(encoding='utf-8', check_for_duplicates=True)
po.metadata = {"Content-Type": "text/plain; charset=utf-8"}
locale_out_dir = path.join(LOCALE_OUT_DIR, locale)
output_path = path.join(locale_out_dir, RELAY_LOCATIONS_PO_FILENAME)
hits = 0
misses = 0
if not path.exists(locale_out_dir):
os.makedirs(locale_out_dir)
for country in countries:
country_name = country.get("name")
country_code = country.get("code")
translated_country_name = country_translator.translate(
locale, country_code)
found_country_translation = translated_country_name is not None
# Default to empty string if no translation was found
if found_country_translation:
hits += 1
else:
translated_country_name = ""
misses += 1
log_message = "{} ({}) -> \"{}\"".format(country_name, country_code,
translated_country_name)
if found_country_translation:
print(c.green(log_message))
else:
print(c.orange(log_message))
# translate country
entry = POEntry(msgid=country_name,
msgstr=translated_country_name,
comment=country_code.upper())
po.append(entry)
# translate cities
cities = country.get("cities")
if cities is None:
print(
c.orange("Skip {} ({}) because no cities were found.".format(
country_name, country_code)))
continue
for city in cities:
city_name = city.get("name")
city_code = city.get("code")
if city_name is None:
raise ValueError("Missing the name field in city record.")
# Make sure to append the US state back to the translated name of the city
if country_code == "us":
split = city_name.rsplit(",", 2)
translated_name = city_translator.translate(
locale, split[0].strip())
if translated_name is not None and len(split) > 1:
translated_name = "{}, {}".format(translated_name,
split[1].strip())
else:
translated_name = city_translator.translate(locale, city_name)
# Default to empty string if no translation was found
found_translation = translated_name is not None
if found_translation:
hits += 1
else:
translated_name = ""
misses += 1
log_message = "{} ({}) -> \"{}\"".format(city_name, city_code,
translated_name)
if found_translation:
print(c.green(log_message))
else:
print(c.orange(log_message))
entry = POEntry(msgid=city_name,
msgstr=translated_name,
comment="{} {}".format(country_code.upper(),
city_code.upper()))
try:
po.append(entry)
except ValueError as err:
print(c.orange("Cannot add an entry: {}".format(err)))
po.save(output_path)
return (hits, misses)
def test_ok(self):
# no markup
test_markup(POEntry(msgid="test string"))
# internal markup
test_markup(POEntry(msgid="<b>test</b> string"))
def test_fail_fuzzy(self):
entry = POEntry(msgid="Source", msgstr="Translation", flags=['fuzzy'])
status = Status()
status.step(entry)
self.assertFalse(fuzzy_validator(status))
def test_unnecessary_markup(self):
self.assertRaises(AssertionError, test_markup,
POEntry(msgid="<b>test string</b>"))
def extract_cities_po():
input_path = get_shape_path("ne_50m_populated_places")
stats = []
for locale in os.listdir(LOCALE_DIR):
locale_dir = path.join(LOCALE_DIR, locale)
locale_out_dir = path.join(LOCALE_OUT_DIR, locale)
if os.path.isdir(locale_dir):
po = POFile(encoding='utf-8', check_for_duplicates=True)
po.metadata = {"Content-Type": "text/plain; charset=utf-8"}
output_path = path.join(locale_out_dir, CITIES_PO_FILENAME)
hits = 0
misses = 0
if not path.exists(locale_out_dir):
os.makedirs(locale_out_dir)
print("Generating {}".format(output_path))
with fiona.open(input_path) as source:
for feat in source:
props = lower_dict_keys(feat["properties"])
if props["pop_max"] >= POPULATION_MAX_FILTER:
name_key = "name_" + map_locale(locale)
name_fallback = "name"
if props.get(name_key) is not None:
translated_name = props.get(name_key)
hits += 1
elif props.get(name_fallback) is not None:
translated_name = props.get(name_fallback)
print(
c.orange("Missing translation for {}".format(
translated_name)))
misses += 1
else:
raise ValueError(
"Cannot find the translation for {}. Probe keys: {}"
.format(locale, (name_key, name_fallback)))
entry = POEntry(msgid=props.get("name"),
msgstr=translated_name)
try:
po.append(entry)
except ValueError as err:
print(
c.orange(
"Cannot add an entry: {}".format(err)))
sort_pofile_entries(po)
po.save(output_path)
print(
c.green("Extracted {} cities to {}".format(
len(po), output_path)))
stats.append((locale, hits, misses))
print_stats_table("Cities translations", stats)