def format_translation(value, language, diff=None, search_match=None,
simple=False, num_plurals=2, unit=None):
"""Nicely formats translation text possibly handling plurals or diff."""
# Split plurals to separate strings
plurals = split_plural(value)
# Show plurals?
if int(num_plurals) <= 1:
plurals = plurals[-1:]
# Newline concatenator
newline = SPACE_NL.format(_('New line'))
# Split diff plurals
if diff is not None:
diff = split_plural(diff)
# Previous message did not have to be a plural
while len(diff) < len(plurals):
diff.append(diff[0])
# We will collect part for each plural
parts = []
for idx, raw_value in enumerate(plurals):
# HTML escape
value = escape(force_text(raw_value))
# Format diff if there is any
value = fmt_diff(value, diff, idx)
# Create span for checks highlights
value = fmt_highlights(raw_value, value, unit)
# Format search term
value = fmt_search(value, search_match)
# Normalize newlines
value = NEWLINES_RE.sub('\n', value)
# Split string
paras = value.split('\n')
# Format whitespace in each paragraph
paras = [fmt_whitespace(p) for p in paras]
# Show label for plural (if there are any)
title = ''
if len(plurals) > 1:
title = language.get_plural_name(idx)
# Join paragraphs
content = mark_safe(newline.join(paras))
parts.append({'title': title, 'content': content})
return {
'simple': simple,
'items': parts,
'language': language,
}
def fmttranslation(value, language=None, diff=None):
'''
Formats translation to show whitespace, plural forms or diff.
'''
# Get language
if language is None:
language = Language.objects.get(code='en')
# Split plurals to separate strings
plurals = split_plural(value)
# Split diff plurals
if diff is not None:
diff = split_plural(diff)
# Previous message did not have to be a plural
while len(diff) < len(plurals):
diff.append(diff[0])
# We will collect part for each plural
parts = []
for idx, value in enumerate(plurals):
# HTML escape
value = escape(force_unicode(value))
# Format diff if there is any
if diff is not None:
diffvalue = escape(force_unicode(diff[idx]))
value = html_diff(diffvalue, value)
# Normalize newlines
value = NEWLINES_RE.sub('\n', value)
# Split string
paras = value.split('\n')
# Format whitespace in each paragraph
paras = [fmt_whitespace(p) for p in paras]
# Show label for plural (if there are any)
if len(plurals) > 1:
value = '<span class="pluraltxt">%s</span><br />' % language.get_plural_label(idx)
else:
value = ''
# Join paragraphs
newline = u'<span class="hlspace" title="%s">↵</span><br />' % _('New line')
value += newline.join(paras)
parts.append(value)
value = '<hr />'.join(parts)
return mark_safe(
'<span lang="%s" dir="%s" class="direction">%s</span>' %
(language.code, language.direction, value)
)
def merge_translations(self, request, store2, overwrite, add_fuzzy, fuzzy,
merge_header):
"""Merge translation unit wise
Needed for template based translations to add new strings.
"""
not_found = 0
skipped = 0
accepted = 0
# Are there any translations to propagate?
# This is just an optimalization to avoid doing that for every unit.
propagate = Translation.objects.filter(
language=self.language,
subproject__project=self.subproject.project).filter(
subproject__allow_translation_propagation=True).exclude(
pk=self.pk).exists()
author = get_author_name(request.user)
# Commit possible prior changes
self.commit_pending(request, author)
# Avoid committing while we're importing
self._skip_commit = True
for set_fuzzy, unit2 in store2.iterate_merge(fuzzy):
try:
unit = self.unit_set.get_unit(unit2)
except Unit.DoesNotExist:
not_found += 1
continue
if unit.translated and not overwrite:
skipped += 1
continue
accepted += 1
unit.translate(request,
split_plural(unit2.get_target()),
add_fuzzy or set_fuzzy,
change_action=Change.ACTION_UPLOAD,
propagate=propagate)
self._skip_commit = False
if accepted > 0:
self.update_stats()
if merge_header:
self.store.merge_header(store2)
self.store.save()
self.git_commit(request,
author,
timezone.now(),
force_commit=True,
sync=True)
return (not_found, skipped, accepted, store2.count_units())
def handle_revert(unit, request, next_unit_url):
revertform = RevertForm(unit, request.GET)
if not revertform.is_valid():
messages.error(request, _("Invalid revert request!"))
return None
change = revertform.cleaned_data["revert_change"]
if not request.user.has_perm("unit.edit", unit):
messages.error(request,
_("Insufficient privileges for saving translations."))
return None
if not change.can_revert():
messages.error(request, _("Can not revert to empty translation!"))
return None
# Store unit
unit.translate(
request.user,
split_plural(change.old),
STATE_FUZZY
if change.action == Change.ACTION_MARKED_EDIT else unit.state,
change_action=Change.ACTION_REVERT,
)
# Redirect to next entry
return HttpResponseRedirect(next_unit_url)
def get_target_plurals(self, plurals=None):
"""Return target plurals in array."""
# Is this plural?
if not self.is_plural:
return [self.target]
# Split plurals
ret = split_plural(self.target)
if plurals is None:
plurals = self.translation.plural.number
# Check if we have expected number of them
if len(ret) == plurals:
return ret
# Pad with empty translations
while len(ret) < plurals:
ret.append("")
# Delete extra plurals
while len(ret) > plurals:
del ret[-1]
return ret
def merge_translations(self, request, store2, overwrite, add_fuzzy, fuzzy):
"""
Merges translation unit wise, needed for template based translations to
add new strings.
"""
ret = False
for set_fuzzy, unit2 in store2.iterate_merge(fuzzy):
try:
unit = self.unit_set.get(
source=unit2.get_source(),
context=unit2.get_context(),
)
except Unit.DoesNotExist:
continue
if unit.translated and not overwrite:
continue
ret = True
unit.translate(request, split_plural(unit2.get_target()), add_fuzzy
or set_fuzzy)
return ret
def get_target_plurals(self):
"""
Returns target plurals in array.
"""
# Is this plural?
if not self.is_plural():
return [self.target]
# Split plurals
ret = split_plural(self.target)
# Check if we have expected number of them
plurals = self.translation.language.nplurals
if len(ret) == plurals:
return ret
# Pad with empty translations
while len(ret) < plurals:
ret.append('')
# Delete extra plurals
while len(ret) > plurals:
del ret[-1]
return ret
def merge_translations(self, request, author, store2, overwrite,
add_fuzzy):
"""
Merges translation unit wise, needed for template based translations to
add new strings.
"""
for unit2 in store2.all_units():
# No translated -> skip
if not unit2.is_translated() or unit2.unit.isheader():
continue
try:
unit = self.unit_set.get(
source=unit2.get_source(),
context=unit2.get_context(),
)
except Unit.DoesNotExist:
continue
unit.translate(
request,
split_plural(unit2.get_target()),
add_fuzzy or unit2.is_fuzzy()
)
def merge_translations(self, request, store2, overwrite, add_fuzzy, fuzzy):
"""
Merges translation unit wise, needed for template based translations to
add new strings.
"""
ret = False
for set_fuzzy, unit2 in store2.iterate_merge(fuzzy):
try:
unit = self.unit_set.get(
source=unit2.get_source(),
context=unit2.get_context(),
)
except Unit.DoesNotExist:
continue
if unit.translated and not overwrite:
continue
ret = True
unit.translate(
request,
split_plural(unit2.get_target()),
add_fuzzy or set_fuzzy
)
return ret
def get_target_plurals(self):
"""
Returns target plurals in array.
"""
# Is this plural?
if not self.is_plural():
return [self.target]
# Split plurals
ret = split_plural(self.target)
# Check if we have expected number of them
plurals = self.translation.language.nplurals
if len(ret) == plurals:
return ret
# Pad with empty translations
while len(ret) < plurals:
ret.append('')
# Delete extra plurals
while len(ret) > plurals:
del ret[-1]
return ret
def merge_translations(self, request, store2, overwrite, add_fuzzy, fuzzy,
merge_header):
"""Merge translation unit wise
Needed for template based translations to add new strings.
"""
not_found = 0
skipped = 0
accepted = 0
author = get_author_name(request.user)
# Commit possible prior changes
self.commit_pending(request, author)
for set_fuzzy, unit2 in store2.iterate_merge(fuzzy):
try:
unit = self.unit_set.get_unit(unit2)
except Unit.DoesNotExist:
not_found += 1
continue
if ((unit.translated and not overwrite)
or (not can_translate(request.user, unit))):
skipped += 1
continue
accepted += 1
# We intentionally avoid propagating:
# - in most cases it's not desired
# - it slows down import considerably
# - it brings locking issues as import is
# executed with lock held and linked repos
# can't obtain the lock
state = STATE_TRANSLATED
if add_fuzzy or set_fuzzy:
state = STATE_FUZZY
unit.translate(request,
split_plural(unit2.get_target()),
state,
change_action=Change.ACTION_UPLOAD,
propagate=False)
if accepted > 0:
self.invalidate_cache()
if merge_header:
self.store.merge_header(store2)
self.store.save()
self.store_hash()
self.git_commit(request,
author,
timezone.now(),
force_commit=True,
sync=True)
return (not_found, skipped, accepted, store2.count_units())
def merge_translations(
self, request, store2, conflicts: str, method: str, fuzzy: str
):
"""Merge translation unit wise.
Needed for template based translations to add new strings.
"""
not_found = 0
skipped = 0
accepted = 0
add_fuzzy = method == "fuzzy"
add_approve = method == "approve"
unit_set = self.unit_set.all()
for set_fuzzy, unit2 in store2.iterate_merge(fuzzy):
try:
unit = unit_set.get_unit(unit2)
except Unit.DoesNotExist:
not_found += 1
continue
state = STATE_TRANSLATED
if add_fuzzy or set_fuzzy:
state = STATE_FUZZY
elif add_approve:
state = STATE_APPROVED
if (
(unit.translated and not conflicts)
or (unit.approved and conflicts != "replace-approved")
or unit.readonly
or (not request.user.has_perm("unit.edit", unit))
or (unit.target == unit2.target and unit.state == state)
):
skipped += 1
continue
accepted += 1
# We intentionally avoid propagating:
# - in most cases it's not desired
# - it slows down import considerably
# - it brings locking issues as import is
# executed with lock held and linked repos
# can't obtain the lock
unit.translate(
request.user,
split_plural(unit2.target),
state,
change_action=Change.ACTION_UPLOAD,
propagate=False,
)
if accepted > 0:
self.invalidate_cache()
request.user.profile.increase_count("translated", accepted)
return (not_found, skipped, accepted, len(list(store2.content_units)))
def handle_add_upload(self, request, store, fuzzy: str = ""):
skipped = 0
accepted = 0
existing = set(self.unit_set.values_list("context", "source"))
units = []
for _set_fuzzy, unit in store.iterate_merge(fuzzy):
if (unit.context, unit.source) in existing:
skipped += 1
continue
units.append((
unit.context,
split_plural(unit.source),
split_plural(unit.target),
))
accepted += 1
self.add_units(request, units)
return (0, skipped, accepted, len(list(store.content_units)))
def merge_translations(self, request, store2, overwrite, add_fuzzy,
fuzzy, merge_header):
"""Merge translation unit wise
Needed for template based translations to add new strings.
"""
not_found = 0
skipped = 0
accepted = 0
# Commit possible prior changes
self.commit_pending(request)
for set_fuzzy, unit2 in store2.iterate_merge(fuzzy):
try:
unit = self.unit_set.get_unit(unit2)
except Unit.DoesNotExist:
not_found += 1
continue
if ((unit.translated and not overwrite)
or (not request.user.has_perm('unit.edit', unit))):
skipped += 1
continue
accepted += 1
# We intentionally avoid propagating:
# - in most cases it's not desired
# - it slows down import considerably
# - it brings locking issues as import is
# executed with lock held and linked repos
# can't obtain the lock
state = STATE_TRANSLATED
if add_fuzzy or set_fuzzy:
state = STATE_FUZZY
unit.translate(
request,
split_plural(unit2.get_target()),
state,
change_action=Change.ACTION_UPLOAD,
propagate=False
)
if accepted > 0:
self.invalidate_cache()
request.user.profile.refresh_from_db()
request.user.profile.translated += accepted
request.user.profile.save(update_fields=['translated'])
if merge_header:
self.store.merge_header(store2)
self.store.save()
self.commit_pending(request)
return (not_found, skipped, accepted, store2.count_units())
def merge_translations(self, request, store2, overwrite, method, fuzzy,
merge_header):
"""Merge translation unit wise
Needed for template based translations to add new strings.
"""
not_found = 0
skipped = 0
accepted = 0
add_fuzzy = (method == 'fuzzy')
add_approve = (method == 'approve')
for set_fuzzy, unit2 in store2.iterate_merge(fuzzy):
try:
unit = self.unit_set.get_unit(unit2)
except Unit.DoesNotExist:
not_found += 1
continue
if ((unit.translated and not overwrite)
or (not request.user.has_perm('unit.edit', unit))):
skipped += 1
continue
accepted += 1
# We intentionally avoid propagating:
# - in most cases it's not desired
# - it slows down import considerably
# - it brings locking issues as import is
# executed with lock held and linked repos
# can't obtain the lock
state = STATE_TRANSLATED
if add_fuzzy or set_fuzzy:
state = STATE_FUZZY
elif add_approve:
state = STATE_APPROVED
unit.translate(request,
split_plural(unit2.get_target()),
state,
change_action=Change.ACTION_UPLOAD,
propagate=False)
if accepted > 0:
self.invalidate_cache()
request.user.profile.refresh_from_db()
request.user.profile.translated += accepted
request.user.profile.save(update_fields=['translated'])
if merge_header:
self.store.merge_header(store2)
self.store.save()
self.commit_pending('upload', request)
return (not_found, skipped, accepted, store2.count_units())
def handle_add_upload(self, request, store, fuzzy: str = ""):
skipped = 0
accepted = 0
existing = set(self.unit_set.values_list("context", "source"))
for _set_fuzzy, unit in store.iterate_merge(fuzzy):
if (unit.context, unit.source) in existing:
skipped += 1
continue
self.add_unit(
request,
unit.context,
split_plural(unit.source),
split_plural(unit.target),
is_batch_update=True,
)
accepted += 1
self.invalidate_cache()
self.component.update_variants()
self.component.sync_terminology()
self.component.update_source_checks()
self.component.run_batched_checks()
return (0, skipped, accepted, len(list(store.content_units)))
def add_unit(self, unit):
output = self.build_unit(unit)
# Propagate source language
if hasattr(output, "setsource"):
output.setsource(output.source, sourcelang=self.source_language.code)
# Location needs to be set prior to ID to avoid overwrite
# on some formats (for example xliff)
for location in unit.location.split():
if location:
output.addlocation(location)
# Store context as context and ID
context = self.string_filter(unit.context)
if context:
output.setcontext(context)
if self.set_id:
output.setid(context)
elif self.set_id:
# Use checksum based ID on formats requiring it
output.setid(unit.checksum)
# Store note
note = self.string_filter(unit.note)
if note:
self.add_note(output, note, origin="developer")
# In Weblate explanation
note = self.string_filter(unit.source_unit.explanation)
if note:
self.add_note(output, note, origin="developer")
# Comments
for comment in unit.unresolved_comments:
self.add_note(output, comment.comment, origin="translator")
# Suggestions
for suggestion in unit.suggestions:
self.add_note(
output,
"Suggested in Weblate: {}".format(
", ".join(split_plural(suggestion.target))
),
origin="translator",
)
# Store flags
if unit.all_flags:
self.store_flags(output, unit.all_flags)
# Store fuzzy flag
if unit.fuzzy:
output.markfuzzy(True)
self.storage.addunit(output)
def fmttranslation(value, language=None, diff=None):
if language is None:
language = Language.objects.get(code="en")
plurals = split_plural(value)
if diff is not None:
diff = split_plural(diff)
parts = []
for idx, value in enumerate(plurals):
value = escape(force_unicode(value))
if diff is not None:
diffvalue = escape(force_unicode(diff[idx]))
value = htmlDiff(diffvalue, value)
value = re.sub(r"\r\n|\r|\n", "\n", value) # normalize newlines
paras = re.split("\n", value)
paras = [fmt_whitespace(p) for p in paras]
if len(plurals) > 1:
value = '<span class="pluraltxt">%s</span><br />' % language.get_plural_label(idx)
else:
value = ""
value += u'<span class="hlspace">↵</span><br />'.join(paras)
parts.append(value)
value = "<hr />".join(parts)
return mark_safe(value)
def add_unit(self, unit):
output = self.storage.UnitClass(
self.handle_plurals(unit.get_source_plurals()))
self.add(output, self.handle_plurals(unit.get_target_plurals()))
# Location needs to be set prior to ID to avoid overwrite
# on some formats (eg. xliff)
for location in unit.location.split():
if location:
output.addlocation(location)
# Store context as context and ID
context = self.string_filter(unit.context)
if context:
output.setcontext(context)
if self.set_id:
output.setid(context)
elif self.set_id:
# Use checksum based ID on formats requiring it
output.setid(unit.checksum)
# Store note
note = self.string_filter(unit.note)
if note:
output.addnote(note, origin='developer')
# In Weblate context
note = self.string_filter(unit.extra_context)
if context:
output.addnote(note, origin='developer')
# Comments
for comment in unit.get_comments():
output.addnote(comment.comment, origin='translator')
# Suggestions
for suggestion in unit.suggestions:
output.addnote(
'Suggested in Weblate: {}'.format(', '.join(
split_plural(suggestion.target))),
origin='translator',
)
# Store flags
if unit.all_flags:
self.store_flags(output, unit.all_flags)
# Store fuzzy flag
if unit.fuzzy:
output.markfuzzy(True)
self.storage.addunit(output)
def fixup_num_words(apps, schema_editor):
db_alias = schema_editor.connection.alias
Unit = apps.get_model("trans", "Unit")
translations = {}
for unit in Unit.objects.using(db_alias).filter(num_words=0):
unit.num_words = len(split_plural(unit.source)[0].split())
unit.save(update_fields=["num_words"])
if unit.translation_id not in translations:
translations[unit.translation_id] = unit.translation
# Invalidate caches
for translation in translations.values():
BaseStats(translation).invalidate()
BaseStats(translation.component).invalidate()
BaseStats(translation.component.project).invalidate()
def accept(self, request, permission="suggestion.accept"):
if not request.user.has_perm(permission, self.unit):
messages.error(request, _("Failed to accept suggestion!"))
return
# Skip if there is no change
if self.unit.target != self.target or self.unit.state < STATE_TRANSLATED:
if self.user and not self.user.is_anonymous:
author = self.user
else:
author = request.user
self.unit.translate(
request.user,
split_plural(self.target),
STATE_TRANSLATED,
author=author,
change_action=Change.ACTION_ACCEPT,
)
# Delete the suggestion
self.delete()
def merge_translations(self, request, author, store2, overwrite,
add_fuzzy):
"""
Merges translation unit wise, needed for template based translations to
add new strings.
"""
for unit2 in store2.all_units():
# No translated -> skip
if not unit2.is_translated() or unit2.unit.isheader():
continue
try:
unit = self.unit_set.get(
source=unit2.get_source(),
context=unit2.get_context(),
)
except Unit.DoesNotExist:
continue
unit.translate(request, split_plural(unit2.get_target()), add_fuzzy
or unit2.is_fuzzy())
def validate_new_unit_data( # noqa: C901
self,
context: str,
source: Union[str, List[str]],
target: Optional[Union[str, List[str]]] = None,
auto_context: bool = False,
extra_flags: Optional[str] = None,
):
extra = {}
if isinstance(source, str):
source = [source]
if isinstance(target, str):
target = [target]
if not self.component.has_template():
extra["source"] = join_plural(source)
if not auto_context and self.unit_set.filter(context=context, **extra).exists():
raise ValidationError(_("This string seems to already exist."))
# Avoid using source translations without a filename
if not self.filename:
try:
translation = self.component.translation_set.exclude(pk=self.pk)[0]
except IndexError:
raise ValidationError(
_("Failed adding string: %s") % _("No translation found.")
)
translation.validate_new_unit_data(
context,
source,
target,
auto_context=auto_context,
extra_flags=extra_flags,
)
return
# Always load a new copy of store
store = self.load_store()
old_units = len(store.all_units)
# Add new unit
store.new_unit(context, source, target, skip_build=True)
# Serialize the content
handle = BytesIOMode("", b"")
# Catch serialization error
try:
store.save_content(handle)
except Exception as error:
raise ValidationError(_("Failed adding string: %s") % error)
handle.seek(0)
# Parse new file (check that it is valid)
try:
newstore = self.load_store(handle)
except Exception as error:
raise ValidationError(_("Failed adding string: %s") % error)
# Verify there is a single unit added
if len(newstore.all_units) != old_units + 1:
raise ValidationError(
_("Failed adding string: %s") % _("Failed to parse new string")
)
# Find newly added unit (it can be on any position), but we assume
# the storage has consistent ordering
unit = None
for pos, current in enumerate(newstore.all_units):
if pos >= old_units or (
current.source != store.all_units[pos].source
and current.context != store.all_units[pos].context
):
unit = current
break
# Verify unit matches data
if unit is None:
raise ValidationError(
_("Failed adding string: %s") % _("Failed to parse new string")
)
created_source = split_plural(unit.source)
if unit.context != context and (
self.component.has_template()
or self.component.file_format_cls.set_context_bilingual
):
raise ValidationError(
{"context": _('Context would be created as "%s"') % unit.context}
)
if created_source != source:
raise ValidationError(
{"source": _("Source would be created as %s") % created_source}
)
def format_translation(value, language, diff=None, search_match=None, simple=False, num_plurals=2, checks=None):
"""
Nicely formats translation text possibly handling plurals or diff.
"""
# Split plurals to separate strings
plurals = split_plural(value)
# Show plurals?
if num_plurals <= 1:
plurals = plurals[:1]
# Newline concatenator
newline = SPACE_NL.format(_("New line"))
# Split diff plurals
if diff is not None:
diff = split_plural(diff)
# Previous message did not have to be a plural
while len(diff) < len(plurals):
diff.append(diff[0])
# We will collect part for each plural
parts = []
for idx, value in enumerate(plurals):
highlights = None
# Find all checks highlight
if checks:
highlights = []
for c in checks:
highlights += c.check_highlight(value, None)
# Sort by order in string
if highlights:
highlights.sort(key=lambda tup: tup[0])
# remove probelmatics ones
for n in xrange(0, len(highlights)):
if n >= len(highlights):
break
elref = highlights[n]
for n2 in xrange(n, len(highlights)):
if n2 >= len(highlights):
break
eltest = highlights[n2]
if eltest[0] >= elref[0] and eltest[0] <= (elref[0] + len(elref[1])):
highlights.pop(n2)
elif eltest[0] > (elref[0] + len(elref[1])):
break
# then transform highlights to escaped html
highlights = [(h[0], escape(force_unicode(h[1]))) for h in highlights]
# HTML escape
value = escape(force_unicode(value))
# Format diff if there is any
if diff is not None:
diffvalue = escape(force_unicode(diff[idx]))
value = html_diff(diffvalue, value)
# Create span for checks highlights
if highlights:
n = 0
for (hidx, htext) in highlights:
p = value.find(htext, n)
if p >= 0:
newpart = u'<span class="hlcheck">{0}</span>'.format(htext)
value = value[:p] + newpart + value[(p + len(htext)) :]
n = p + len(newpart)
# Format search term
if search_match:
# Since the search ignored case, we need to highlight any
# combination of upper and lower case we find. This is too
# advanced for str.replace().
caseless = re.compile(re.escape(search_match), re.IGNORECASE)
for variation in re.findall(caseless, value):
value = re.sub(caseless, u'<span class="hlmatch">{0}</span>'.format(variation), value)
# Normalize newlines
value = NEWLINES_RE.sub("\n", value)
# Split string
paras = value.split("\n")
# Format whitespace in each paragraph
paras = [fmt_whitespace(p) for p in paras]
# Show label for plural (if there are any)
title = ""
if len(plurals) > 1:
title = language.get_plural_label(idx)
# Join paragraphs
content = mark_safe(newline.join(paras))
parts.append({"title": title, "content": content})
return {"simple": simple, "items": parts, "language": language}
def get_source_plurals(self):
"""
Returns source plurals in array.
"""
return split_plural(self.source)
def format_translation(value, language=None, diff=None, search_match=None,
simple=False):
"""
Nicely formats translation text possibly handling plurals or diff.
"""
# Get language
if language is None:
language = Language.objects.get_default()
# Split plurals to separate strings
plurals = split_plural(value)
# Newline concatenator
newline = u'<span class="hlspace" title="{0}">↵</span><br />'.format(
_('New line')
)
# Split diff plurals
if diff is not None:
diff = split_plural(diff)
# Previous message did not have to be a plural
while len(diff) < len(plurals):
diff.append(diff[0])
# We will collect part for each plural
parts = []
for idx, value in enumerate(plurals):
# HTML escape
value = escape(force_unicode(value))
# Format diff if there is any
if diff is not None:
diffvalue = escape(force_unicode(diff[idx]))
value = html_diff(diffvalue, value)
# Format search term
if search_match:
# Since the search ignored case, we need to highlight any
# combination of upper and lower case we find. This is too
# advanced for str.replace().
caseless = re.compile(re.escape(search_match), re.IGNORECASE)
for variation in re.findall(caseless, value):
value = re.sub(
caseless,
u'<span class="hlmatch">{0}</span>'.format(variation),
value,
)
# Normalize newlines
value = NEWLINES_RE.sub('\n', value)
# Split string
paras = value.split('\n')
# Format whitespace in each paragraph
paras = [fmt_whitespace(p) for p in paras]
# Show label for plural (if there are any)
title = ''
if len(plurals) > 1:
title = language.get_plural_label(idx)
# Join paragraphs
content = mark_safe(newline.join(paras))
parts.append({'title': title, 'content': content})
return {
'simple': simple,
'items': parts,
'language': language,
}
def format_translation(
value,
language,
plural=None,
diff=None,
search_match=None,
simple=False,
wrap=False,
num_plurals=2,
unit=None,
match="search",
):
"""Nicely formats translation text possibly handling plurals or diff."""
# Split plurals to separate strings
plurals = split_plural(value)
if plural is None:
plural = language.plural
# Show plurals?
if int(num_plurals) <= 1:
plurals = plurals[-1:]
# Newline concatenator
newline = SPACE_NL.format(gettext("New line"))
# Split diff plurals
if diff is not None:
diff = split_plural(diff)
# Previous message did not have to be a plural
while len(diff) < len(plurals):
diff.append(diff[0])
# We will collect part for each plural
parts = []
has_content = False
for idx, raw_value in enumerate(plurals):
# HTML escape
value = raw_value
# Content of the Copy to clipboard button
copy = escape(value)
# Format diff if there is any
value = fmt_diff(value, diff, idx)
# Create span for checks highlights
value = fmt_highlights(raw_value, value, unit)
# Format search term
value = fmt_search(value, search_match, match)
# Normalize newlines
value = NEWLINES_RE.sub("\n", value)
# Split string
paras = value.split("\n")
# Format whitespace in each paragraph
paras = [fmt_whitespace(p) for p in paras]
# Show label for plural (if there are any)
title = ""
if len(plurals) > 1:
title = plural.get_plural_name(idx)
# Join paragraphs
content = mark_safe(newline.join(paras))
parts.append({"title": title, "content": content, "copy": copy})
has_content |= bool(content)
return {
"simple": simple,
"wrap": wrap,
"items": parts,
"language": language,
"unit": unit,
"has_content": has_content,
}
def get_source_plurals(self):
'''
Returns source plurals in array.
'''
return split_plural(self.source)
def format_translation(value, language, plural=None, diff=None,
search_match=None, simple=False, num_plurals=2,
unit=None, match='search'):
"""Nicely formats translation text possibly handling plurals or diff."""
# Split plurals to separate strings
plurals = split_plural(value)
if plural is None:
plural = language.plural
# Show plurals?
if int(num_plurals) <= 1:
plurals = plurals[-1:]
# Newline concatenator
newline = SPACE_NL.format(_('New line'))
# Split diff plurals
if diff is not None:
diff = split_plural(diff)
# Previous message did not have to be a plural
while len(diff) < len(plurals):
diff.append(diff[0])
# We will collect part for each plural
parts = []
for idx, raw_value in enumerate(plurals):
# HTML escape
value = escape(force_text(raw_value))
# Format diff if there is any
value = fmt_diff(value, diff, idx)
# Create span for checks highlights
value = fmt_highlights(raw_value, value, unit)
# Format search term
value = fmt_search(value, search_match, match)
# Normalize newlines
value = NEWLINES_RE.sub('\n', value)
# Split string
paras = value.split('\n')
# Format whitespace in each paragraph
paras = [fmt_whitespace(p) for p in paras]
# Show label for plural (if there are any)
title = ''
if len(plurals) > 1:
title = plural.get_plural_name(idx)
# Join paragraphs
content = mark_safe(newline.join(paras))
parts.append({'title': title, 'content': content})
return {
'simple': simple,
'items': parts,
'language': language,
}
def update_from_unit(self, unit, pos, created):
"""Update Unit from ttkit unit."""
component = self.translation.component
self.is_batch_update = True
# Get unit attributes
try:
location = unit.locations
flags = unit.flags
target = unit.target
self.check_valid(split_plural(target))
source = unit.source
self.check_valid(split_plural(source))
context = unit.context
self.check_valid([context])
note = unit.notes
previous_source = unit.previous_source
content_hash = unit.content_hash
except Exception as error:
report_error(cause="Unit update error")
self.translation.component.handle_parse_error(
error, self.translation)
# Ensure we track source string for bilingual
if not self.translation.is_source:
source_info = component.get_source(
self.id_hash,
source=source,
target=source,
context=context,
content_hash=calculate_hash(source, context),
position=0,
location=location,
flags=flags,
)
self.extra_context = source_info.extra_context
self.extra_flags = source_info.extra_flags
self.__dict__["source_info"] = source_info
# Calculate state
state = self.get_unit_state(unit, flags)
self.original_state = self.get_unit_state(unit, None)
# Has source changed
same_source = source == self.source and context == self.context
# Monolingual files handling (without target change)
if not created and unit.template is not None and target == self.target:
if not same_source and state >= STATE_TRANSLATED:
if self.previous_source == self.source and self.fuzzy:
# Source change was reverted
previous_source = ""
state = STATE_TRANSLATED
else:
# Store previous source and fuzzy flag for monolingual
if previous_source == "":
previous_source = self.source
state = STATE_FUZZY
elif self.state in (STATE_FUZZY, STATE_APPROVED):
# We should keep calculated flags if translation was
# not changed outside
previous_source = self.previous_source
state = self.state
# Update checks on fuzzy update or on content change
same_target = target == self.target
same_state = state == self.state and flags == self.flags
# Check if we actually need to change anything
# pylint: disable=too-many-boolean-expressions
if (location == self.location and flags == self.flags and same_source
and same_target and same_state and note == self.note
and pos == self.position and content_hash == self.content_hash
and previous_source == self.previous_source):
return
# Store updated values
self.position = pos
self.location = location
self.flags = flags
self.source = source
self.target = target
self.state = state
self.context = context
self.note = note
self.content_hash = content_hash
self.previous_source = previous_source
self.update_priority(save=False)
# Sanitize number of plurals
if self.is_plural():
self.target = join_plural(self.get_target_plurals())
if created:
unit_pre_create.send(sender=self.__class__, unit=self)
# Save into database
self.save(
force_insert=created,
same_content=same_source and same_target,
same_state=same_state,
)
# Track updated sources for source checks
if self.translation.is_template:
component.updated_sources[self.id_hash] = self
# Update unit labels
if not self.translation.is_source:
self.labels.set(self.source_info.labels.all())
# Indicate source string change
if not same_source and previous_source:
Change.objects.create(
unit=self,
action=Change.ACTION_SOURCE_CHANGE,
old=previous_source,
target=self.source,
)
def update_from_unit(self, unit, pos, created): # noqa: C901
"""Update Unit from ttkit unit."""
translation = self.translation
component = translation.component
self.is_batch_update = True
self.source_updated = True
# Get unit attributes
try:
location = unit.locations
flags = unit.flags
source = unit.source
self.check_valid(split_plural(source))
if not translation.is_template and translation.is_source:
# Load target from source string for bilingual source translations
target = source
else:
target = unit.target
self.check_valid(split_plural(target))
context = unit.context
self.check_valid([context])
note = unit.notes
previous_source = unit.previous_source
except Exception as error:
report_error(cause="Unit update error")
translation.component.handle_parse_error(error, translation)
# Ensure we track source string for bilingual, this can not use
# Unit.is_source as that depends on source_unit attribute, which
# we set here
old_source_unit = self.source_unit
if not translation.is_source:
self.update_source_unit(component, source, context, pos, note,
location, flags)
# Calculate state
state = self.get_unit_state(unit, flags)
original_state = self.get_unit_state(unit, None)
# Has source changed
same_source = source == self.source and context == self.context
# Monolingual files handling (without target change)
if (not created and state != STATE_READONLY
and unit.template is not None and target == self.target):
if not same_source and state in (STATE_TRANSLATED, STATE_APPROVED):
if self.previous_source == self.source and self.fuzzy:
# Source change was reverted
previous_source = ""
state = STATE_TRANSLATED
else:
# Store previous source and fuzzy flag for monolingual
if previous_source == "":
previous_source = self.source
state = STATE_FUZZY
elif self.state in (STATE_FUZZY, STATE_APPROVED):
# We should keep calculated flags if translation was
# not changed outside
previous_source = self.previous_source
state = self.state
original_state = self.original_state
# Update checks on fuzzy update or on content change
same_target = target == self.target
same_state = state == self.state and flags == self.flags
same_metadata = (location == self.location and note == self.note
and pos == self.position)
same_data = (not created and same_source and same_target and same_state
and original_state == self.original_state
and flags == self.flags
and previous_source == self.previous_source
and self.source_unit == old_source_unit
and old_source_unit is not None)
# Check if we actually need to change anything
if same_data and same_metadata:
return
# Store updated values
self.original_state = original_state
self.position = pos
self.location = location
self.flags = flags
self.source = source
self.target = target
self.state = state
self.context = context
self.note = note
self.previous_source = previous_source
self.update_priority(save=False)
# Metadata update only, these do not trigger any actions in Weblate and
# are display only
if same_data and not same_metadata:
self.save(same_content=True, only_save=True)
return
# Sanitize number of plurals
if self.is_plural:
self.target = join_plural(self.get_target_plurals())
if created:
unit_pre_create.send(sender=self.__class__, unit=self)
# Save into database
self.save(
force_insert=created,
same_content=same_source and same_target,
run_checks=not same_source or not same_target or not same_state,
)
# Track updated sources for source checks
if translation.is_template:
component.updated_sources[self.id] = self
# Indicate source string change
if not same_source and previous_source:
Change.objects.create(
unit=self,
action=Change.ACTION_SOURCE_CHANGE,
old=previous_source,
target=self.source,
)
# Update translation memory if needed
if (self.state >= STATE_TRANSLATED
and (not translation.is_source or component.intermediate)
and (created or not same_source or not same_target)):
transaction.on_commit(
lambda: handle_unit_translation_change.delay(self.id))
def format_translation(value,
language,
diff=None,
search_match=None,
simple=False,
num_plurals=2,
checks=None):
"""
Nicely formats translation text possibly handling plurals or diff.
"""
# Split plurals to separate strings
plurals = split_plural(value)
# Show plurals?
if num_plurals <= 1:
plurals = plurals[:1]
# Newline concatenator
newline = SPACE_NL.format(_('New line'))
# Split diff plurals
if diff is not None:
diff = split_plural(diff)
# Previous message did not have to be a plural
while len(diff) < len(plurals):
diff.append(diff[0])
# We will collect part for each plural
parts = []
for idx, value in enumerate(plurals):
highlights = None
# Find all checks highlight
if checks:
highlights = []
for c in checks:
highlights += c.check_highlight(value, None)
#Sort by order in string
if highlights: highlights.sort(key=lambda tup: tup[0])
#remove probelmatics ones
for n in xrange(0, len(highlights)):
if n >= len(highlights): break
elref = highlights[n]
for n2 in xrange(n, len(highlights)):
if n2 >= len(highlights): break
eltest = highlights[n2]
if eltest[0] >= elref[0] and eltest[0] <= (elref[0] +
len(elref[1])):
highlights.pop(n2)
elif eltest[0] > (elref[0] + len(elref[1])):
break
#then transform highlights to escaped html
highlights = [(h[0], escape(force_unicode(h[1])))
for h in highlights]
# HTML escape
value = escape(force_unicode(value))
# Format diff if there is any
if diff is not None:
diffvalue = escape(force_unicode(diff[idx]))
value = html_diff(diffvalue, value)
# Create span for checks highlights
if highlights:
n = 0
for (hidx, htext) in highlights:
p = value.find(htext, n)
if p >= 0:
newpart = u'<span class="hlcheck">{0}</span>'.format(htext)
value = value[:p] + newpart + value[(p + len(htext)):]
n = p + len(newpart)
# Format search term
if search_match:
# Since the search ignored case, we need to highlight any
# combination of upper and lower case we find. This is too
# advanced for str.replace().
caseless = re.compile(re.escape(search_match), re.IGNORECASE)
for variation in re.findall(caseless, value):
value = re.sub(
caseless,
u'<span class="hlmatch">{0}</span>'.format(variation),
value,
)
# Normalize newlines
value = NEWLINES_RE.sub('\n', value)
# Split string
paras = value.split('\n')
# Format whitespace in each paragraph
paras = [fmt_whitespace(p) for p in paras]
# Show label for plural (if there are any)
title = ''
if len(plurals) > 1:
title = language.get_plural_label(idx)
# Join paragraphs
content = mark_safe(newline.join(paras))
parts.append({'title': title, 'content': content})
return {
'simple': simple,
'items': parts,
'language': language,
}
def get_source_plurals(self):
'''
Returns source plurals in array.
'''
return split_plural(self.source)
def merge_translations(self, request, store2, overwrite, add_fuzzy,
fuzzy, merge_header):
"""Merge translation unit wise
Needed for template based translations to add new strings.
"""
not_found = 0
skipped = 0
accepted = 0
# Are there any translations to propagate?
# This is just an optimalization to avoid doing that for every unit.
propagate = Translation.objects.filter(
language=self.language,
subproject__project=self.subproject.project
).filter(
subproject__allow_translation_propagation=True
).exclude(
pk=self.pk
).exists()
author = get_author_name(request.user)
# Commit possible prior changes
self.commit_pending(request, author)
# Avoid committing while we're importing
self._skip_commit = True
for set_fuzzy, unit2 in store2.iterate_merge(fuzzy):
try:
unit = self.unit_set.get_unit(unit2)
except Unit.DoesNotExist:
not_found += 1
continue
if unit.translated and not overwrite:
skipped += 1
continue
accepted += 1
unit.translate(
request,
split_plural(unit2.get_target()),
add_fuzzy or set_fuzzy,
change_action=Change.ACTION_UPLOAD,
propagate=propagate
)
self._skip_commit = False
if accepted > 0:
self.update_stats()
if merge_header:
self.store.merge_header(store2)
self.store.save()
self.git_commit(
request, author, timezone.now(),
force_commit=True, sync=True
)
return (not_found, skipped, accepted, store2.count_units())
def update_from_unit(self, unit, pos, created):
"""Update Unit from ttkit unit."""
translation = self.translation
component = translation.component
self.is_batch_update = True
self.source_updated = True
# Get unit attributes
try:
location = unit.locations
flags = unit.flags
source = unit.source
self.check_valid(split_plural(source))
if not translation.is_template and translation.is_source:
# Load target from source string for bilingual source translations
target = source
else:
target = unit.target
self.check_valid(split_plural(target))
context = unit.context
self.check_valid([context])
note = unit.notes
previous_source = unit.previous_source
content_hash = unit.content_hash
except Exception as error:
report_error(cause="Unit update error")
translation.component.handle_parse_error(error, translation)
# Ensure we track source string for bilingual, this can not use
# Unit.is_source as that depends on source_unit attribute, which
# we set here
old_source_unit = self.source_unit
if not translation.is_source:
source_unit = component.get_source(
self.id_hash,
create={
"source": source,
"target": source,
"context": context,
"content_hash": calculate_hash(source, context),
"position": pos,
"note": note,
"location": location,
"flags": flags,
},
)
if (not source_unit.source_updated
and not component.has_template() and
(pos != source_unit.position
or location != source_unit.location
or flags != source_unit.flags or note != source_unit.note)):
source_unit.position = pos
source_unit.source_updated = True
source_unit.location = location
source_unit.flags = flags
source_unit.note = note
source_unit.save(
update_fields=["position", "location", "flags", "note"],
same_content=True,
run_checks=False,
)
self.source_unit = source_unit
# Calculate state
state = self.get_unit_state(unit, flags)
self.original_state = self.get_unit_state(unit, None)
# Has source changed
same_source = source == self.source and context == self.context
# Monolingual files handling (without target change)
if (not created and state != STATE_READONLY
and unit.template is not None and target == self.target):
if not same_source and state in (STATE_TRANSLATED, STATE_APPROVED):
if self.previous_source == self.source and self.fuzzy:
# Source change was reverted
previous_source = ""
state = STATE_TRANSLATED
else:
# Store previous source and fuzzy flag for monolingual
if previous_source == "":
previous_source = self.source
state = STATE_FUZZY
elif self.state in (STATE_FUZZY, STATE_APPROVED):
# We should keep calculated flags if translation was
# not changed outside
previous_source = self.previous_source
state = self.state
# Update checks on fuzzy update or on content change
same_target = target == self.target
same_state = state == self.state and flags == self.flags
# Check if we actually need to change anything
# pylint: disable=too-many-boolean-expressions
if (not created and same_source and same_target and same_state
and location == self.location and flags == self.flags
and note == self.note and pos == self.position
and content_hash == self.content_hash
and previous_source == self.previous_source
and self.source_unit == old_source_unit
and old_source_unit is not None):
return
# Store updated values
self.position = pos
self.location = location
self.flags = flags
self.source = source
self.target = target
self.state = state
self.context = context
self.note = note
self.content_hash = content_hash
self.previous_source = previous_source
self.update_priority(save=False)
# Sanitize number of plurals
if self.is_plural:
self.target = join_plural(self.get_target_plurals())
if created:
unit_pre_create.send(sender=self.__class__, unit=self)
# Save into database
self.save(
force_insert=created,
same_content=same_source and same_target,
run_checks=not same_source or not same_target or not same_state,
)
# Track updated sources for source checks
if translation.is_template:
component.updated_sources[self.id_hash] = self
# Indicate source string change
if not same_source and previous_source:
Change.objects.create(
unit=self,
action=Change.ACTION_SOURCE_CHANGE,
old=previous_source,
target=self.source,
)
def fmttranslation(value, language=None, diff=None, search_match=None):
'''
Formats translation to show whitespace, plural forms or diff.
'''
# Get language
if language is None:
language = Language.objects.get_default()
# Split plurals to separate strings
plurals = split_plural(value)
# Split diff plurals
if diff is not None:
diff = split_plural(diff)
# Previous message did not have to be a plural
while len(diff) < len(plurals):
diff.append(diff[0])
# We will collect part for each plural
parts = []
for idx, value in enumerate(plurals):
# HTML escape
value = escape(force_unicode(value))
# Format diff if there is any
if diff is not None:
diffvalue = escape(force_unicode(diff[idx]))
value = html_diff(diffvalue, value)
# Format search term
if search_match is not None:
# Since the search ignored case, we need to highlight any
# combination of upper and lower case we find. This is too
# advanced for str.replace().
caseless = re.compile(re.escape(search_match), re.IGNORECASE)
for variation in re.findall(caseless, value):
value = re.sub(
caseless,
'<span class="hlmatch">%s</span>' % (variation),
value,
)
# Normalize newlines
value = NEWLINES_RE.sub('\n', value)
# Split string
paras = value.split('\n')
# Format whitespace in each paragraph
paras = [fmt_whitespace(p) for p in paras]
# Show label for plural (if there are any)
if len(plurals) > 1:
value = '<span class="pluraltxt">%s</span><br />' % (
language.get_plural_label(idx))
else:
value = ''
# Join paragraphs
newline = u'<span class="hlspace" title="%s">↵</span><br />' % (
_('New line'))
value += newline.join(paras)
parts.append(value)
value = '<hr />'.join(parts)
return mark_safe('<span lang="%s" dir="%s" class="direction">%s</span>' %
(language.code, language.direction, value))
def format_translation(
value,
language,
plural=None,
diff=None,
search_match=None,
simple: bool = False,
wrap: bool = False,
noformat: bool = False,
num_plurals=2,
unit=None,
match="search",
glossary=None,
):
"""Nicely formats translation text possibly handling plurals or diff."""
# Split plurals to separate strings
plurals = split_plural(value)
if plural is None:
plural = language.plural
# Show plurals?
if int(num_plurals) <= 1:
plurals = plurals[-1:]
# Split diff plurals
if diff is not None:
diff = split_plural(diff)
# Previous message did not have to be a plural
while len(diff) < len(plurals):
diff.append(diff[0])
terms = defaultdict(list)
for term in glossary or []:
terms[term.source].append(term)
# We will collect part for each plural
parts = []
has_content = False
for idx, text in enumerate(plurals):
formatter = Formatter(idx, text, unit, terms, diff, search_match,
match)
formatter.parse()
# Show label for plural (if there are any)
title = ""
if len(plurals) > 1:
title = plural.get_plural_name(idx)
# Join paragraphs
content = formatter.format()
parts.append({
"title": title,
"content": content,
"copy": escape(text)
})
has_content |= bool(content)
return {
"simple": simple,
"noformat": noformat,
"wrap": wrap,
"items": parts,
"language": language,
"unit": unit,
"has_content": has_content,
}
def get_source_plurals(self):
"""Return source plurals in array."""
return split_plural(self.source)
def fmttranslation(value, language=None, diff=None, search_match=None):
'''
Formats translation to show whitespace, plural forms or diff.
'''
# Get language
if language is None:
language = Language.objects.get_default()
# Split plurals to separate strings
plurals = split_plural(value)
# Split diff plurals
if diff is not None:
diff = split_plural(diff)
# Previous message did not have to be a plural
while len(diff) < len(plurals):
diff.append(diff[0])
# We will collect part for each plural
parts = []
for idx, value in enumerate(plurals):
# HTML escape
value = escape(force_unicode(value))
# Format diff if there is any
if diff is not None:
diffvalue = escape(force_unicode(diff[idx]))
value = html_diff(diffvalue, value)
# Format search term
if search_match is not None:
# Since the search ignored case, we need to highlight any
# combination of upper and lower case we find. This is too
# advanced for str.replace().
caseless = re.compile(re.escape(search_match), re.IGNORECASE)
for variation in re.findall(caseless, value):
value = re.sub(
caseless,
'<span class="hlmatch">%s</span>' % (variation),
value,
)
# Normalize newlines
value = NEWLINES_RE.sub('\n', value)
# Split string
paras = value.split('\n')
# Format whitespace in each paragraph
paras = [fmt_whitespace(p) for p in paras]
# Show label for plural (if there are any)
if len(plurals) > 1:
value = '<span class="pluraltxt">%s</span><br />' % (
language.get_plural_label(idx)
)
else:
value = ''
# Join paragraphs
newline = u'<span class="hlspace" title="%s">↵</span><br />' % (
_('New line')
)
value += newline.join(paras)
parts.append(value)
value = '<hr />'.join(parts)
return mark_safe(
'<span lang="%s" dir="%s" class="direction">%s</span>' %
(language.code, language.direction, value)
)
