def translate( self, user, new_target, new_state, change_action=None, propagate: bool = True, author=None, ): """ Store new translation of a unit. Propagation is currently disabled on import. """ # Fetch current copy from database and lock it for update self.old_unit = Unit.objects.select_for_update(**get_nokey_args()).get( pk=self.pk) # Handle simple string units if isinstance(new_target, str): new_target = [new_target] # Apply autofixes if not self.translation.is_template: new_target, self.fixups = fix_target(new_target, self) # Update unit and save it self.target = join_plural(new_target) not_empty = bool(max(new_target)) # Newlines fixup if "dos-eol" in self.all_flags: self.target = NEWLINES.sub("\r\n", self.target) if not_empty: self.state = new_state else: self.state = STATE_EMPTY self.original_state = self.state saved = self.save_backend(user, change_action=change_action, propagate=propagate, author=author) # Enforced checks can revert the state to needs editing (fuzzy) if (self.state >= STATE_TRANSLATED and self.translation.component.enforced_checks and self.all_checks_names & set(self.translation.component.enforced_checks)): self.state = self.original_state = STATE_FUZZY self.save(run_checks=False, same_content=True, update_fields=["state"]) if (propagate and user and self.target != self.old_unit.target and self.state >= STATE_TRANSLATED): transaction.on_commit( lambda: handle_unit_translation_change.delay(self.id, user.id)) return saved
def process_others(self, source): """Perform automatic translation based on other components.""" kwargs = { "translation__language": self.translation.language, "state__gte": STATE_TRANSLATED, } source_language = self.translation.component.source_language exclude = {} if source: component = Component.objects.get(id=source) if (not component.project.contribute_shared_tm and not component.project != self.translation.component.project ) or component.source_language != source_language: raise PermissionDenied() kwargs["translation__component"] = component else: project = self.translation.component.project kwargs["translation__component__project"] = project kwargs["translation__component__source_language"] = source_language exclude["translation"] = self.translation sources = Unit.objects.filter(**kwargs) if exclude: sources = sources.exclude(**exclude) # Fetch translations translations = { source: (state, target) for source, state, target in sources.filter( source__in=self.get_units().values("source")).values_list( "source", "state", "target") } # We need to skip mode (suggestions) filtering here as SELECT FOR UPDATE # cannot be used with JOIN units = (self.get_units(False).filter( source__in=translations.keys()).select_for_update( **get_nokey_args())) self.progress_steps = len(units) for pos, unit in enumerate(units): # Get update state, target = translations[unit.source] self.set_progress(pos) # No save if translation is same or unit does not exist if unit.state == state and unit.target == target: continue # Copy translation self.update(unit, state, target) self.post_process()
def process_mt(self, engines, threshold): """Perform automatic translation based on machine translation.""" translations = self.fetch_mt(engines, int(threshold)) # Adjust total number to show correct progress offset = self.progress_steps / 2 self.progress_steps = offset + len(translations) with transaction.atomic(): # Perform the translation for pos, unit in enumerate( Unit.objects.filter(id__in=translations.keys()).prefetch(). select_for_update(**get_nokey_args())): # Copy translation self.update(unit, self.target_state, translations[unit.pk]) self.set_progress(offset + pos) self.post_process()
def search_replace(request, project, component=None, lang=None): obj, unit_set, context = parse_url(request, project, component, lang) form = ReplaceForm(request.POST) if not form.is_valid(): messages.error(request, _("Failed to process form!")) show_form_errors(request, form) return redirect(obj) search_text = form.cleaned_data["search"] replacement = form.cleaned_data["replacement"] matching = unit_set.filter(target__contains=search_text) updated = 0 if matching.exists(): confirm = ReplaceConfirmForm(matching, request.POST) limited = False if matching.count() > 300: matching = matching.order_by("id")[:250] limited = True if not confirm.is_valid(): for unit in matching: unit.replacement = unit.target.replace(search_text, replacement) context.update({ "matching": matching, "search_query": search_text, "replacement": replacement, "form": form, "limited": limited, "confirm": ReplaceConfirmForm(matching), }) return render(request, "replace.html", context) matching = confirm.cleaned_data["units"] with transaction.atomic(): for unit in matching.select_for_update(**get_nokey_args()): if not request.user.has_perm("unit.edit", unit): continue unit.translate( request.user, unit.target.replace(search_text, replacement), unit.state, change_action=Change.ACTION_REPLACE, ) updated += 1 import_message( request, updated, _("Search and replace completed, no strings were updated."), ngettext( "Search and replace completed, %d string was updated.", "Search and replace completed, %d strings were updated.", updated, ), ) return redirect(obj)
def bulk_perform( user, unit_set, query, target_state, add_flags, remove_flags, add_labels, remove_labels, project, components=None, ): matching = unit_set.search(query, project=project).prefetch() if components is None: components = Component.objects.filter(id__in=matching.values_list( "translation__component_id", flat=True)) target_state = int(target_state) add_flags = Flags(add_flags) remove_flags = Flags(remove_flags) update_source = add_flags or remove_flags or add_labels or remove_labels updated = 0 for component in components: with transaction.atomic(), component.lock(): component.commit_pending("bulk edit", user) component_units = matching.filter(translation__component=component) source_unit_ids = set() if target_state == -1: # Only fetch source unit ids here source_unit_ids = set( component_units.values_list("source_unit_id", flat=True)) else: update_unit_ids = [] source_units = [] # Generate changes for state change for unit in component_units.select_for_update( **get_nokey_args()): source_unit_ids.add(unit.source_unit_id) if ((user is None or user.has_perm("unit.edit", unit)) and target_state != unit.state and unit.state in EDITABLE_STATES): # Create change object for edit, update is done outside the loop unit.generate_change(user, user, Change.ACTION_BULK_EDIT, check_new=False) updated += 1 update_unit_ids.append(unit.pk) if unit.is_source: source_units.append(unit) # Bulk update state Unit.objects.filter(pk__in=update_unit_ids).update( pending=True, state=target_state) # Fire source_change event in bulk for source units for unit in source_units: # The change is already done in the database, we # need it here to recalculate state of translation # units unit.is_bulk_edit = True unit.pending = True unit.state = target_state unit.source_unit_save() if update_source and (user is None or user.has_perm("source.edit", component)): # Perform changes on the source units source_units = Unit.objects.filter( pk__in=source_unit_ids).prefetch() if add_labels or remove_labels: source_units = source_units.prefetch_related("labels") for source_unit in source_units.select_for_update( **get_nokey_args()): changed = False if add_flags or remove_flags: flags = Flags(source_unit.extra_flags) flags.merge(add_flags) flags.remove(remove_flags) new_flags = flags.format() if source_unit.extra_flags != new_flags: source_unit.is_bulk_edit = True source_unit.extra_flags = new_flags source_unit.save(update_fields=["extra_flags"]) changed = True if add_labels: source_unit.is_bulk_edit = True source_unit.labels.add(*add_labels) changed = True if remove_labels: source_unit.is_bulk_edit = True source_unit.labels.remove(*remove_labels) changed = True if changed: updated += 1 component.invalidate_cache() return updated
def select_for_update(self): return super().select_for_update(**get_nokey_args())
def check_sync(self, force=False, request=None, change=None): # noqa: C901 """Check whether database is in sync with git and possibly updates.""" if change is None: change = Change.ACTION_UPDATE if request is None: user = None else: user = request.user # Check if we're not already up to date if not self.revision: self.reason = "new file" elif self.revision != self.get_git_blob_hash(): self.reason = "content changed" elif force: self.reason = "check forced" else: self.reason = "" return self.log_info("processing %s, %s", self.filename, self.reason) # List of updated units (used for cleanup and duplicates detection) updated = {} try: store = self.store translation_store = None # Store plural plural = store.get_plural(self.language) if plural != self.plural: self.plural = plural self.save(update_fields=["plural"]) # Was there change? self.was_new = 0 # Select all current units for update dbunits = { unit.id_hash: unit for unit in self.unit_set.select_for_update(**get_nokey_args()) } # Process based on intermediate store if available if self.component.intermediate: translation_store = store store = self.load_store(force_intermediate=True) for pos, unit in enumerate(store.content_units): # Use translation store if exists and if it contains the string if translation_store is not None: try: translated_unit, created = translation_store.find_unit( unit.context) if translated_unit and not created: unit = translated_unit else: # Patch unit to have matching source unit.source = translated_unit.source except UnitNotFound: pass id_hash = unit.id_hash # Check for possible duplicate units if id_hash in updated: newunit = updated[id_hash] self.log_warning( "duplicate string to translate: %s (%s)", newunit, repr(newunit.source), ) Change.objects.create( unit=newunit, action=Change.ACTION_DUPLICATE_STRING, user=user, author=user, ) self.component.trigger_alert( "DuplicateString", language_code=self.language.code, source=newunit.source, unit_pk=newunit.pk, ) continue self.sync_unit(dbunits, updated, id_hash, unit, pos + 1) except FileParseError as error: self.log_warning("skipping update due to parse error: %s", error) return # Delete stale units stale = set(dbunits) - set(updated) if stale: self.unit_set.filter(id_hash__in=stale).delete() self.component.needs_cleanup = True # We should also do cleanup on source strings tracking objects # Update revision and stats self.store_hash() # Store change entry Change.objects.create(translation=self, action=change, user=user, author=user) # Invalidate keys cache transaction.on_commit(self.invalidate_keys) self.log_info("updating completed") # Use up to date list as prefetch for source if self.is_source: self.component.preload_sources(updated)