def normaliseAllPoFiles(filesGlob):
if prefs['normalisePoFiles']:
fileIndex = 0
for fileName in sorted(filesGlob):
fileIndex = fileIndex + 1
poFile = polib.pofile(fileName)
poFile.wrapwidth = 1024 # avoid re-wrapping
poFile.check_for_duplicates = True
print("\rNormalising translation files..." +
" {0:.0%}".format(float(fileIndex) /
len(sorted(filesGlob)))),
sys.stdout.flush()
for cacheEntry in sorted(occurrencesCache, key=lambda r: r[0]):
matchedEntries = [
e for e in poFile if e.msgid == polib.unescape(cacheEntry)
]
if len(matchedEntries) == 0 and len(
occurrencesCache[cacheEntry]) > 0:
try:
newEntry = polib.POEntry(
msgid=polib.unescape(cacheEntry),
msgstr="",
occurrences=occurrencesCache[cacheEntry],
tcomment="normalised")
poFile.append(newEntry)
except:
pass
elif len(matchedEntries) == 1 and matchedEntries[0].obsolete:
# matchedEntries[0].obsolete = False
pass
poFile.save(fileName + prefs['newFileExt'])
def dump_po(project, locale):
"""Update .po (gettext) files from database."""
locale_paths = get_locale_paths(project, locale)
for path in locale_paths:
po = polib.pofile(path)
date = datetime.datetime(1, 1, 1)
newest = Translation()
relative_path = get_relative_path(path, locale)
resource = Resource.objects.filter(project=project, path=relative_path)
entities = Entity.objects.filter(resource=resource, obsolete=False)
for entity in entities:
entry = po.find(polib.unescape(smart_text(entity.string)))
if entry:
if not entry.msgid_plural:
translation = get_translation(entity=entity, locale=locale)
if translation.string != '':
entry.msgstr = polib.unescape(translation.string)
if translation.date > date:
date = translation.date
newest = translation
if ('fuzzy' in entry.flags and not translation.fuzzy):
entry.flags.remove('fuzzy')
else:
for i in range(0, 6):
if i < (locale.nplurals or 1):
translation = get_translation(
entity=entity, locale=locale, plural_form=i)
if translation.string != '':
entry.msgstr_plural[unicode(i)] = \
polib.unescape(translation.string)
if translation.date > date:
date = translation.date
newest = translation
if ('fuzzy' in entry.flags and
not translation.fuzzy):
entry.flags.remove('fuzzy')
# Remove obsolete plural forms if exist
else:
if unicode(i) in entry.msgstr_plural:
del entry.msgstr_plural[unicode(i)]
# Update PO metadata
if newest.id:
po.metadata['PO-Revision-Date'] = newest.date
if newest.user:
po.metadata['Last-Translator'] = '%s <%s>' \
% (newest.user.first_name, newest.user.email)
po.metadata['Language'] = locale.code
po.metadata['X-Generator'] = 'Pontoon'
if locale.nplurals:
po.metadata['Plural-Forms'] = 'nplurals=%s; plural=%s;' \
% (str(locale.nplurals), locale.plural_rule)
po.save()
log.debug("File updated: " + path)
def validate(self, old, new):
old = unescape(old)
new = unescape(new)
for c in self.bracket_chars:
if new.count(c) != old.count(c):
raise ValidationError(
_("Translation string doesn't contain the same "
"number of '%s' as the source string." % c))
def validate(self, old, new):
old = unescape(old)
new = unescape(new)
for email in self.emails.findall(old):
if email not in new:
raise ValidationError(
_("The following email is either missing from the"
" translation or has been translated: '%s'." % email))
def validate(self, old, new):
old = unescape(old)
new = unescape(new)
for email in self.emails.findall(old):
if email not in new:
raise ValidationError(
_("The following email is either missing from the"
" translation or has been translated: '%s'." % email)
)
def validate(self, old, new):
old = unescape(old)
new = unescape(new)
for c in self.bracket_chars:
if new.count(c) != old.count(c):
raise ValidationError(
_("Translation string doesn't contain the same "
"number of '%s' as the source string." % c)
)
def validate(self, old, new):
old = unescape(old)
new = unescape(new)
old_matches = list(self.printf_re.finditer(old))
new_matches = list(self.printf_re.finditer(new))
if len(old_matches) != len(new_matches):
raise ValidationError("The number of arguments seems to differ "
"between the source string and the translation."
)
def validate(self, old, new):
old = unescape(old)
new = unescape(new)
old_matches = list(self.printf_re.finditer(old))
new_matches = list(self.printf_re.finditer(new))
if len(old_matches) != len(new_matches):
raise ValidationError(
"The number of arguments seems to differ "
"between the source string and the translation.")
def validate(self, old, new):
old = unescape(old)
new = unescape(new)
for num in self.numbers.findall(old):
if num not in new:
num = num.replace('.', ',', 1)
if num not in new:
raise ValidationError(
_("Number %s is in the source string but not "
"in the translation." % num))
def validate(self, source_trans, target_trans):
"""Check, if all printf-format expressions in the source translation
are in the target translation, too.
We are interested in the conversion specifier and the positional
key (if any). See ``printf(3)`` for details.
So, we check, whether *every* conversion specifier found in
the source translation exists in the target translation, too.
Additionally, if there are positional keys, whether those
found in the source translation exist in the target translation,
as well.
We raise a ``ValidationError``, whenever one of the
conditions is not met.
Args:
source_trans: The source translation.
target_trans: The target translation.
Raises:
ValidationError, in case the translation is not valid.
"""
source_trans = unescape(source_trans)
target_trans = unescape(target_trans)
source_matches = list(self.printf_re.finditer(source_trans))
target_matches = list(self.printf_re.finditer(target_trans))
# We could use just one list comprehension:
#
# target_data = [
# (pattern.group('type'), pattern.group('key'))
# for pattern in target_matches
# ]
# target_specifiers, target_keys = map(
# list, zip(*target_data)
# ) or [[], []]
#
# but that would probably be less efficient, since target_matches
# should ususally have 0 - 5 elements, and much less readable.
# So, we do it in two steps.
target_specifiers = [pat.group('type') for pat in target_matches]
target_keys = [pattern.group('key') for pattern in target_matches]
for pattern in source_matches:
key = pattern.group('key')
if key not in target_keys:
msg = "The expression '%s' is not present in the translation."
raise ValidationError( _(msg % pattern.group(0)))
conversion_specifier = pattern.group('type')
try:
target_specifiers.remove(conversion_specifier)
except ValueError:
msg = "The expression '%s' is not present in the translation."
raise ValidationError( _(msg % pattern.group(0)))
def validate(self, source_trans, target_trans):
"""Check, if all printf-format expressions in the source translation
are in the target translation, too.
We are interested in the conversion specifier and the positional
key (if any). See ``printf(3)`` for details.
So, we check, whether *every* conversion specifier found in
the source translation exists in the target translation, too.
Additionally, if there are positional keys, whether those
found in the source translation exist in the target translation,
as well.
We raise a ``ValidationError``, whenever one of the
conditions is not met.
Args:
source_trans: The source translation.
target_trans: The target translation.
Raises:
ValidationError, in case the translation is not valid.
"""
source_trans = unescape(source_trans)
target_trans = unescape(target_trans)
source_matches = list(self.printf_re.finditer(source_trans))
target_matches = list(self.printf_re.finditer(target_trans))
# We could use just one list comprehension:
#
# target_data = [
# (pattern.group('type'), pattern.group('key'))
# for pattern in target_matches
# ]
# target_specifiers, target_keys = map(
# list, zip(*target_data)
# ) or [[], []]
#
# but that would probably be less efficient, since target_matches
# should ususally have 0 - 5 elements, and much less readable.
# So, we do it in two steps.
target_specifiers = [pat.group('type') for pat in target_matches]
target_keys = [pattern.group('key') for pattern in target_matches]
for pattern in source_matches:
key = pattern.group('key')
if key not in target_keys:
msg = "The expression '%s' is not present in the translation."
raise ValidationError(_(msg % pattern.group(0)))
conversion_specifier = pattern.group('type')
try:
target_specifiers.remove(conversion_specifier)
except ValueError:
msg = "The expression '%s' is not present in the translation."
raise ValidationError(_(msg % pattern.group(0)))
def validate(self, old, new):
old = unescape(old)
new = unescape(new)
old_has_newline = old[-1] == '\n'
new_has_newline = new[-1] == '\n'
if old_has_newline != new_has_newline:
if old_has_newline:
msg = _("Translation must end with a newline (\\n)")
else:
msg = _("Translation should not end with a newline (\\n)")
raise ValidationError(msg)
def validate(self, old, new):
old = unescape(old)
new = unescape(new)
old_has_newline = old[-1] == '\n'
new_has_newline = new[-1] == '\n'
if old_has_newline != new_has_newline:
if old_has_newline:
msg = _("Translation must end with a newline (\\n)")
else:
msg = _("Translation should not end with a newline (\\n)")
raise ValidationError(msg)
def validate(self, old, new):
old = unescape(old)
new = unescape(new)
for num in self.numbers.findall(old):
if num not in new:
num = num.replace('.', ',', 1)
if num not in new:
raise ValidationError(
_("Number %s is in the source string but not "
"in the translation." % num)
)
def _save_translation(source_string, translations, target_language, user):
"""Save a translation string to the database.
This functions handle a signle source entity translation
(could be pluralized).
Currently, the function only returns warning strings.
There is no message for success.
Args:
source_string: A Translation object of the string in the source
language.
translations: A (rule, string) tuple.
target_language: The language the string is translated to.
user: The translator.
Returns:
A list if strings to display to the user.
Raises:
An LotteBadRequestError exception in case of errors.
"""
source_id = source_string.pk
resource = source_string.source_entity.resource
source_language = resource.source_language
warnings = []
for rule, target_string in translations.items():
rule = target_language.get_rule_num_from_name(rule)
if rule != 5:
# fetch correct source string for plural rule
try:
source_string = Translation.objects.get(
source_entity=source_string.source_entity,
language=source_language, rule=rule
)
except Translation.DoesNotExist:
# target language has extra plural forms
pass
ss = unescape(source_string.string)
tr = unescape(target_string)
# check for errors
try:
for ErrorValidator in create_error_validators(resource.i18n_type):
v = ErrorValidator(source_language, target_language, rule)
v(ss, tr)
except ValidationError, e:
raise LotteBadRequestError(e.message)
# check for warnings
for WarningValidator in create_warning_validators(resource.i18n_type):
v = WarningValidator(source_language, target_language, rule)
try:
v(ss, tr)
except ValidationError, e:
warnings.append(e.message)
def test_unescape(self):
"""
Tests the unescape function.
"""
self.assertEqual(
polib.unescape('\\\\t and \\\\n and \\\\r and \\\\" and \\\\\\\\'),
'\\t and \\n and \\r and \\" and \\\\')
def solve(new_file: str, old_file: str, arguments):
""" Translates single file. """
lines = read_lines(old_file)
for line in lines:
line.msgstr = polib.unescape(
translate(polib.escape(line.msgid), arguments))
print(f"Translated {lines.percent_translated()}% of the lines.")
save_lines(new_file, lines)
def test_unescape(self):
"""
Tests the unescape function.
"""
self.assertEqual(
polib.unescape('\\\\t and \\\\n and \\\\r and \\\\" and \\\\\\\\'),
'\\t and \\n and \\r and \\" and \\\\'
)
def parse_msg(lines):
" Extracts msgid or msgstr from lines."
if len(lines) < 1:
return ''
i = lines[0].find('"')
if i < 0:
return ''
msg = lines[0][i:].strip('"')
for i in range(1, len(lines)):
j = lines[i].find('"')
if j < 0:
return ''
msg = msg + lines[i][j:].strip('"')
return polib.unescape(msg)
def parse_msg(lines):
" Extracts msgid or msgstr from lines."
if len(lines) < 1:
return ''
i = lines[0].find('"')
if i < 0:
return ''
msg = lines[0][i:].strip('"')
for i in range(1, len(lines)):
j = lines[i].find('"')
if j < 0:
return ''
msg = msg + lines[i][j:].strip('"')
return polib.unescape(msg)
def read(glos, filename):
fp = open(filename, 'rb')
word = ''
defi = ''
msgstr = False
for line in fp:
if not line:
if word:
glos.addEntry(word, defi)
word = ''
defi = ''
break
line = line.strip()
if not line:
continue
if line.startswith('#'):
continue
if line.startswith('msgid '):
if word:
glos.addEntry(word, defi)
word = ''
defi = ''
word = unescape(line[6:])
msgstr = False
elif line.startswith('msgstr '):
if msgstr:
log.error('msgid omitted!')
defi = unescape(line[7:])
msgstr = True
else:
if msgstr:
defi += unescape(line)
else:
word += unescape(line)
if word:
glos.addEntry(word, defi)
def create_terminal_quest_assets(podir, lang):
print("Creating terminal quest assets")
pofile = os.path.join(podir, lang, "assets.po")
po = polib.pofile(pofile)
with tempfile.TemporaryDirectory() as tmpdirname:
for entry in po:
for filename in entry.occurrences:
f = open(os.path.join(tmpdirname, filename[0]), "w")
value = entry.msgstr if entry.translated() else entry.msgid
f.write(polib.unescape(value))
f.close
# SCP directory
copy_terminal_quest_assets(tmpdirname, lang)
# We don't need the assets.po file anymore
os.remove(pofile)
line = NSIWorkingFile.readline()
lineNo += 1
while line != '':
line = removeEscapedNewLine(line)
end = line.find('"')
if end != -1: #If we found the closing character, append
value += line[:end].lstrip()
break
else: #If not, append and continue
value += line.lstrip()
line = NSIWorkingFile.readline()
lineNo += 1
# Remove whitespace and new lines
value = value.strip("\t\n")
value = polib.unescape(value)
if not value in LangStringCache:
LangStringCache[value] = []
# Note down our file and line number
LangStringCache[value].append([options.input, lineNo])
if not value in LangStringLabels:
LangStringLabels[value] = []
# Note down our label
LangStringLabels[value].append(label)
line = NSIWorkingFile.readline()
lineNo += 1
# Now, we loop through our cache and build PO entries for each
# We use PO comment field to store our NSIS labels, so we can decode it back later
def validate(self, old, new):
new = unescape(new)
if len(new.strip()) == 0:
raise ValidationError(
_("Translation string only contains whitespaces."))
line = NSIWorkingFile.readline()
lineNo += 1
while line != '':
line = removeEscapedNewLine(line)
end = line.find('"')
if end != -1: #If we found the closing character, append
value += line[:end].lstrip()
break
else: #If not, append and continue
value += line.lstrip()
line=NSIWorkingFile.readline()
lineNo += 1
# Remove whitespace and new lines
value = value.strip("\t\n")
value = polib.unescape ( value )
if not value in LangStringCache:
LangStringCache[value] = []
# Note down our file and line number
LangStringCache[value].append([options.input,lineNo])
if not value in LangStringLabels:
LangStringLabels[value] = []
# Note down our label
LangStringLabels[value].append(label)
line=NSIWorkingFile.readline()
lineNo += 1
# Now, we loop through our cache and build PO entries for each
# We use PO comment field to store our NSIS labels, so we can decode it back later
if os.path.isfile(filename):
print(filename + " already exists, we skip it")
continue
context = entry.comment
currentNews = {}
if news.get(filename):
currentNews = news.get(filename)
if "title" in context:
currentNews['title'] = entry.msgstr
currentNews['originalFilename'] = originalFilename
else:
currentNews[polib.unescape(entry.msgid)] = polib.unescape(entry.msgstr)
news[filename] = currentNews
# for all news we have in the pot file, we create the corresponding html
# translated file
for currentNews in news:
if not 'title' in news[currentNews] or news[currentNews]['title'] == "":
print("Skip news", currentNews)
continue
print("Creating", currentNews)
# Read in the file
with open(news[currentNews]['originalFilename'], "r", encoding="utf-8") as originalFile:
fileData = originalFile.read()
def validate(self, old, new):
new = unescape(new)
if len(new.strip()) == 0:
raise ValidationError(
_("Translation string only contains whitespaces.")
)
