def translate(
self,
user,
new_target,
new_state,
change_action=None,
propagate: bool = True,
author=None,
):
"""
Store new translation of a unit.
Propagation is currently disabled on import.
"""
# Fetch current copy from database and lock it for update
self.old_unit = Unit.objects.select_for_update(**get_nokey_args()).get(
pk=self.pk)
# Handle simple string units
if isinstance(new_target, str):
new_target = [new_target]
# Apply autofixes
if not self.translation.is_template:
new_target, self.fixups = fix_target(new_target, self)
# Update unit and save it
self.target = join_plural(new_target)
not_empty = bool(max(new_target))
# Newlines fixup
if "dos-eol" in self.all_flags:
self.target = NEWLINES.sub("\r\n", self.target)
if not_empty:
self.state = new_state
else:
self.state = STATE_EMPTY
self.original_state = self.state
saved = self.save_backend(user,
change_action=change_action,
propagate=propagate,
author=author)
# Enforced checks can revert the state to needs editing (fuzzy)
if (self.state >= STATE_TRANSLATED
and self.translation.component.enforced_checks
and self.all_checks_names
& set(self.translation.component.enforced_checks)):
self.state = self.original_state = STATE_FUZZY
self.save(run_checks=False,
same_content=True,
update_fields=["state"])
if (propagate and user and self.target != self.old_unit.target
and self.state >= STATE_TRANSLATED):
transaction.on_commit(
lambda: handle_unit_translation_change.delay(self.id, user.id))
return saved
def process_others(self, source):
"""Perform automatic translation based on other components."""
kwargs = {
"translation__language": self.translation.language,
"state__gte": STATE_TRANSLATED,
}
source_language = self.translation.component.source_language
exclude = {}
if source:
component = Component.objects.get(id=source)
if (not component.project.contribute_shared_tm and
not component.project != self.translation.component.project
) or component.source_language != source_language:
raise PermissionDenied()
kwargs["translation__component"] = component
else:
project = self.translation.component.project
kwargs["translation__component__project"] = project
kwargs["translation__component__source_language"] = source_language
exclude["translation"] = self.translation
sources = Unit.objects.filter(**kwargs)
if exclude:
sources = sources.exclude(**exclude)
# Fetch translations
translations = {
source: (state, target)
for source, state, target in sources.filter(
source__in=self.get_units().values("source")).values_list(
"source", "state", "target")
}
# We need to skip mode (suggestions) filtering here as SELECT FOR UPDATE
# cannot be used with JOIN
units = (self.get_units(False).filter(
source__in=translations.keys()).select_for_update(
**get_nokey_args()))
self.progress_steps = len(units)
for pos, unit in enumerate(units):
# Get update
state, target = translations[unit.source]
self.set_progress(pos)
# No save if translation is same or unit does not exist
if unit.state == state and unit.target == target:
continue
# Copy translation
self.update(unit, state, target)
self.post_process()
def process_mt(self, engines, threshold):
"""Perform automatic translation based on machine translation."""
translations = self.fetch_mt(engines, int(threshold))
# Adjust total number to show correct progress
offset = self.progress_steps / 2
self.progress_steps = offset + len(translations)
with transaction.atomic():
# Perform the translation
for pos, unit in enumerate(
Unit.objects.filter(id__in=translations.keys()).prefetch().
select_for_update(**get_nokey_args())):
# Copy translation
self.update(unit, self.target_state, translations[unit.pk])
self.set_progress(offset + pos)
self.post_process()
def search_replace(request, project, component=None, lang=None):
obj, unit_set, context = parse_url(request, project, component, lang)
form = ReplaceForm(request.POST)
if not form.is_valid():
messages.error(request, _("Failed to process form!"))
show_form_errors(request, form)
return redirect(obj)
search_text = form.cleaned_data["search"]
replacement = form.cleaned_data["replacement"]
matching = unit_set.filter(target__contains=search_text)
updated = 0
if matching.exists():
confirm = ReplaceConfirmForm(matching, request.POST)
limited = False
if matching.count() > 300:
matching = matching.order_by("id")[:250]
limited = True
if not confirm.is_valid():
for unit in matching:
unit.replacement = unit.target.replace(search_text,
replacement)
context.update({
"matching": matching,
"search_query": search_text,
"replacement": replacement,
"form": form,
"limited": limited,
"confirm": ReplaceConfirmForm(matching),
})
return render(request, "replace.html", context)
matching = confirm.cleaned_data["units"]
with transaction.atomic():
for unit in matching.select_for_update(**get_nokey_args()):
if not request.user.has_perm("unit.edit", unit):
continue
unit.translate(
request.user,
unit.target.replace(search_text, replacement),
unit.state,
change_action=Change.ACTION_REPLACE,
)
updated += 1
import_message(
request,
updated,
_("Search and replace completed, no strings were updated."),
ngettext(
"Search and replace completed, %d string was updated.",
"Search and replace completed, %d strings were updated.",
updated,
),
)
return redirect(obj)
def bulk_perform(
user,
unit_set,
query,
target_state,
add_flags,
remove_flags,
add_labels,
remove_labels,
project,
components=None,
):
matching = unit_set.search(query, project=project).prefetch()
if components is None:
components = Component.objects.filter(id__in=matching.values_list(
"translation__component_id", flat=True))
target_state = int(target_state)
add_flags = Flags(add_flags)
remove_flags = Flags(remove_flags)
update_source = add_flags or remove_flags or add_labels or remove_labels
updated = 0
for component in components:
with transaction.atomic(), component.lock():
component.commit_pending("bulk edit", user)
component_units = matching.filter(translation__component=component)
source_unit_ids = set()
if target_state == -1:
# Only fetch source unit ids here
source_unit_ids = set(
component_units.values_list("source_unit_id", flat=True))
else:
update_unit_ids = []
source_units = []
# Generate changes for state change
for unit in component_units.select_for_update(
**get_nokey_args()):
source_unit_ids.add(unit.source_unit_id)
if ((user is None or user.has_perm("unit.edit", unit))
and target_state != unit.state
and unit.state in EDITABLE_STATES):
# Create change object for edit, update is done outside the loop
unit.generate_change(user,
user,
Change.ACTION_BULK_EDIT,
check_new=False)
updated += 1
update_unit_ids.append(unit.pk)
if unit.is_source:
source_units.append(unit)
# Bulk update state
Unit.objects.filter(pk__in=update_unit_ids).update(
pending=True, state=target_state)
# Fire source_change event in bulk for source units
for unit in source_units:
# The change is already done in the database, we
# need it here to recalculate state of translation
# units
unit.is_bulk_edit = True
unit.pending = True
unit.state = target_state
unit.source_unit_save()
if update_source and (user is None
or user.has_perm("source.edit", component)):
# Perform changes on the source units
source_units = Unit.objects.filter(
pk__in=source_unit_ids).prefetch()
if add_labels or remove_labels:
source_units = source_units.prefetch_related("labels")
for source_unit in source_units.select_for_update(
**get_nokey_args()):
changed = False
if add_flags or remove_flags:
flags = Flags(source_unit.extra_flags)
flags.merge(add_flags)
flags.remove(remove_flags)
new_flags = flags.format()
if source_unit.extra_flags != new_flags:
source_unit.is_bulk_edit = True
source_unit.extra_flags = new_flags
source_unit.save(update_fields=["extra_flags"])
changed = True
if add_labels:
source_unit.is_bulk_edit = True
source_unit.labels.add(*add_labels)
changed = True
if remove_labels:
source_unit.is_bulk_edit = True
source_unit.labels.remove(*remove_labels)
changed = True
if changed:
updated += 1
component.invalidate_cache()
return updated
def select_for_update(self):
return super().select_for_update(**get_nokey_args())
def check_sync(self, force=False, request=None, change=None): # noqa: C901
"""Check whether database is in sync with git and possibly updates."""
if change is None:
change = Change.ACTION_UPDATE
if request is None:
user = None
else:
user = request.user
# Check if we're not already up to date
if not self.revision:
self.reason = "new file"
elif self.revision != self.get_git_blob_hash():
self.reason = "content changed"
elif force:
self.reason = "check forced"
else:
self.reason = ""
return
self.log_info("processing %s, %s", self.filename, self.reason)
# List of updated units (used for cleanup and duplicates detection)
updated = {}
try:
store = self.store
translation_store = None
# Store plural
plural = store.get_plural(self.language)
if plural != self.plural:
self.plural = plural
self.save(update_fields=["plural"])
# Was there change?
self.was_new = 0
# Select all current units for update
dbunits = {
unit.id_hash: unit
for unit in self.unit_set.select_for_update(**get_nokey_args())
}
# Process based on intermediate store if available
if self.component.intermediate:
translation_store = store
store = self.load_store(force_intermediate=True)
for pos, unit in enumerate(store.content_units):
# Use translation store if exists and if it contains the string
if translation_store is not None:
try:
translated_unit, created = translation_store.find_unit(
unit.context)
if translated_unit and not created:
unit = translated_unit
else:
# Patch unit to have matching source
unit.source = translated_unit.source
except UnitNotFound:
pass
id_hash = unit.id_hash
# Check for possible duplicate units
if id_hash in updated:
newunit = updated[id_hash]
self.log_warning(
"duplicate string to translate: %s (%s)",
newunit,
repr(newunit.source),
)
Change.objects.create(
unit=newunit,
action=Change.ACTION_DUPLICATE_STRING,
user=user,
author=user,
)
self.component.trigger_alert(
"DuplicateString",
language_code=self.language.code,
source=newunit.source,
unit_pk=newunit.pk,
)
continue
self.sync_unit(dbunits, updated, id_hash, unit, pos + 1)
except FileParseError as error:
self.log_warning("skipping update due to parse error: %s", error)
return
# Delete stale units
stale = set(dbunits) - set(updated)
if stale:
self.unit_set.filter(id_hash__in=stale).delete()
self.component.needs_cleanup = True
# We should also do cleanup on source strings tracking objects
# Update revision and stats
self.store_hash()
# Store change entry
Change.objects.create(translation=self,
action=change,
user=user,
author=user)
# Invalidate keys cache
transaction.on_commit(self.invalidate_keys)
self.log_info("updating completed")
# Use up to date list as prefetch for source
if self.is_source:
self.component.preload_sources(updated)
