def id_hash(self): """Return hash of source string, used for quick lookup. We use siphash as it is fast and works well for our purpose. """ if self.template is None: return calculate_hash(self.source, self.context) return calculate_hash(None, self.context)
def test_hash(self): """Ensure hash is not changing""" text = 'Message' text_hash = calculate_hash(None, text) self.assertEqual( text_hash, calculate_hash(None, text) )
def translate_cache_key(self, source, language, text): if not self.cache_translations: return None return 'mt:{}:{}:{}'.format( self.mtid, calculate_hash(source, language), calculate_hash(None, text), )
def test_hash_unicode(self): """Ensure hash works for unicode""" text = 'Příšerně žluťoučký kůň úpěl ďábelské ódy' text_hash = calculate_hash(None, text) self.assertEqual( text_hash, calculate_hash(None, text) )
def test_hash_context(self): """Ensure hash works with context""" text = 'Message' context = 'Context' text_hash = calculate_hash(context, text) self.assertEqual( text_hash, calculate_hash(context, text) )
def get_id_hash(self): """Return hash of source string, used for quick lookup. We use siphash as it is fast and works well for our purpose. """ if self.id_hash is None: if self.template is None: self.id_hash = calculate_hash( self.get_source(), self.get_context() ) else: self.id_hash = calculate_hash( None, self.get_context() ) return self.id_hash
def test_checksum(self): """Hash to checksum conversion""" text_hash = calculate_hash(None, 'Message') checksum = hash_to_checksum(text_hash) self.assertEqual( text_hash, checksum_to_hash(checksum) )
def calculate_id_hash(apps, schema_editor): Unit = apps.get_model('trans', 'Unit') Comment = apps.get_model('trans', 'Comment') Check = apps.get_model('trans', 'Check') Suggestion = apps.get_model('trans', 'Suggestion') Source = apps.get_model('trans', 'Source') content_models = (Comment, Check, Suggestion) units = Unit.objects.prefetch_related( 'translation', 'translation__language', 'translation__subproject', 'translation__subproject__project', ) for unit in units.iterator(): unit.content_hash = calculate_hash(unit.source, unit.context) if unit.translation.subproject.template: unit.id_hash = calculate_hash(None, unit.context) else: unit.id_hash = unit.content_hash unit.save(update_fields=['content_hash', 'id_hash']) for model in content_models: model.objects.filter( (Q(language=unit.translation.language) | Q(language=None)) & Q(project=unit.translation.subproject.project) & Q(contentsum=unit.contentsum) ).update( content_hash=unit.content_hash ) Source.objects.filter( subproject=unit.translation.subproject, checksum=unit.checksum ).update( id_hash=unit.id_hash ) # Remove stale instances for model in content_models: model.objects.filter(content_hash=0).delete() Source.objects.filter(id_hash=0).delete()
def get_content_hash(self): """Return hash of source string and context, used for quick lookup.""" if self.template is None: return self.get_id_hash() if self.content_hash is None: self.content_hash = calculate_hash( self.get_source(), self.get_context() ) return self.content_hash
def content_hash(self): """Return hash of source string and context, used for quick lookup.""" if self.template is None: return self.id_hash return calculate_hash(self.source, self.context)
def get_target_hash(self): return calculate_hash(None, self.target)
def migrate_glossaries(apps, schema_editor): # noqa: C901 Project = apps.get_model("trans", "Project") Language = apps.get_model("lang", "Language") db_alias = schema_editor.connection.alias projects = Project.objects.using(db_alias).all() total = len(projects) processed = 0 for processed, project in enumerate(projects): component_slugs = set(project.component_set.values_list("slug", flat=True)) percent = int(100 * processed / total) print(f"Migrating glossaries {percent}% [{processed}/{total}]...{project.name}") glossaries = project.glossary_set.all() try: license = project.component_set.exclude(license="").values_list( "license", flat=True )[0] except IndexError: license = "" for glossary in glossaries: if len(glossaries) == 1: name = "Glossary" slug = "glossary" else: name = f"Glossary: {glossary.name}" slug = f"glossary-{slugify(glossary.name)}" base_name = name base_slug = slug # Create component attempts = 0 while True: if slug not in component_slugs: component = create_glossary(project, name, slug, glossary, license) component_slugs.add(slug) break attempts += 1 name = f"{base_name} - {attempts}" slug = f"{base_slug}-{attempts}" repo_path = os.path.join(settings.DATA_DIR, "vcs", project.slug, slug) # Create VCS repository repo = LocalRepository.from_files(repo_path, {}) # Migrate links component.links.set(glossary.links.all()) # Create source translation source_translation = component.translation_set.create( language=glossary.source_language, check_flags="read-only", filename="", plural=glossary.source_language.plural_set.filter(source=0)[0], language_code=glossary.source_language.code, ) source_units = {} # Get list of languages languages = ( Language.objects.filter(term__glossary=glossary) .exclude(pk=glossary.source_language.pk) .distinct() ) # Migrate ters for language in languages: base_filename = f"{language.code}.tbx" filename = os.path.join(repo_path, base_filename) # Create translation object translation = component.translation_set.create( language=language, plural=language.plural_set.filter(source=0)[0], filename=base_filename, language_code=language.code, ) # Create store file TBXFormat.create_new_file(filename, language.code, "") store = TBXFormat( filename, language_code=language.code, source_language=glossary.source_language.code, ) id_hashes = set() for position, term in enumerate( glossary.term_set.filter(language=language) ): source = valid_chars_only(term.source) target = valid_chars_only(term.target) context = "" # Store to the file id_hash = calculate_hash(source, context) offset = 0 while id_hash in id_hashes: offset += 1 context = str(offset) id_hash = calculate_hash(source, context) id_hashes.add(id_hash) if id_hash not in source_units: source_units[id_hash] = source_translation.unit_set.create( context=context, source=source, target=source, state=STATE_READONLY, position=position, num_words=len(source.split()), id_hash=id_hash, ) source_units[id_hash].source_unit = source_units[id_hash] source_units[id_hash].save() store.new_unit(context, source, target) # Migrate database unit = translation.unit_set.create( context=context, source=source, target=target, state=STATE_TRANSLATED, position=position, num_words=len(source.split()), id_hash=id_hash, source_unit=source_units[id_hash], ) # Adjust history entries to include unit details, # language and project should be already set term.change_set.update( unit=unit, translation=translation, component=component, ) store.save() # Update translation hash translation.revision = repo.get_object_hash(filename) translation.save(update_fields=["revision"]) # Commit files with repo.lock: repo.execute(["add", repo_path]) if repo.needs_commit(): repo.commit("Migrate glossary content") if total: print(f"Migrating glossaries completed [{total}/{total}]")
def save_backend(self, request, propagate=True, gen_change=True, change_action=None, user=None): """ Stores unit to backend. Optional user parameters defines authorship of a change. """ # For case when authorship specified, use user from request if user is None or user.is_anonymous: user = request.user # Update lock timestamp self.update_lock(request, user, change_action) # Commit possible previous changes by other author self.translation.commit_pending(request, get_author_name(user)) # Return if there was no change # We have to explicitly check for fuzzy flag change on monolingual # files, where we handle it ourselves without storing to backend if (self.old_unit.fuzzy == self.fuzzy and self.old_unit.target == self.target): # Propagate if we should if propagate: self.propagate(request, change_action) return False # Propagate to other projects # This has to be done before changing source/content_hash for template if propagate: self.propagate(request, change_action) if self.translation.is_template(): self.source = self.target self.content_hash = calculate_hash(self.source, self.context) # Unit is pending for write self.pending = True # Update translated flag (not fuzzy and at least one transltion) self.translated = (not self.fuzzy and bool(max(self.get_target_plurals()))) # Save updated unit to database self.save(backend=True) # Update translation stats old_translated = self.translation.translated if change_action != Change.ACTION_UPLOAD: self.translation.update_stats() self.translation.store_hash() # Notify subscribed users about new translation notify_new_translation(self, self.old_unit, user) # Update user stats user.profile.translated += 1 user.profile.save() # Generate Change object for this change if gen_change: self.generate_change(request, user, self.old_unit, change_action) # Force commiting on completing translation if (old_translated < self.translation.translated and self.translation.translated == self.translation.total): self.translation.commit_pending(request) Change.objects.create(translation=self.translation, action=Change.ACTION_COMPLETE, user=user, author=user) # Update related source strings if working on a template if self.translation.is_template(): self.update_source_units(self.old_unit.source) return True
def test_hash_unicode(self): """Ensure hash works for unicode.""" text = "Příšerně žluťoučký kůň úpěl ďábelské ódy" text_hash = calculate_hash(text) self.assertEqual(text_hash, -4296353750398394478) self.assertEqual(text_hash, calculate_hash(text))
def test_hash(self): """Ensure hash is not changing""" text = 'Message' text_hash = calculate_hash(None, text) self.assertEqual(text_hash, calculate_hash(None, text))
def test_checksum(self): """Hash to checksum conversion""" text_hash = calculate_hash(None, 'Message') checksum = hash_to_checksum(text_hash) self.assertEqual(text_hash, checksum_to_hash(checksum))
def test_hash_context(self): """Ensure hash works with context""" text = 'Message' context = 'Context' text_hash = calculate_hash(context, text) self.assertEqual(text_hash, calculate_hash(context, text))
def content_hash(self): if self.translation.component.template: return calculate_hash(self.source, self.context) return self.id_hash
def save_backend(self, request, propagate=True, change_action=None, user=None): """ Stores unit to backend. Optional user parameters defines authorship of a change. This should be always called in a trasaction with updated unit locked for update. """ # For case when authorship specified, use user from request if user is None or (user.is_anonymous and request): user = request.user # Commit possible previous changes on this unit if self.pending: change_author = self.get_last_content_change(request)[0] if change_author.id != user.id: self.translation.commit_pending('pending unit', request) # Propagate to other projects # This has to be done before changing source/content_hash for template if propagate: self.propagate(request, change_action) # Return if there was no change # We have to explicitly check for fuzzy flag change on monolingual # files, where we handle it ourselves without storing to backend if (self.old_unit.state == self.state and self.old_unit.target == self.target): return False if self.translation.is_template: self.source = self.target self.content_hash = calculate_hash(self.source, self.context) # Unit is pending for write self.pending = True # Update translated flag (not fuzzy and at least one translation) translation = bool(max(self.get_target_plurals())) if self.state >= STATE_TRANSLATED and not translation: self.state = STATE_EMPTY elif self.state == STATE_EMPTY and translation: self.state = STATE_TRANSLATED # Save updated unit to database self.save() # Run source checks self.source_info.run_checks(unit=self) # Generate Change object for this change self.generate_change(request, user, change_action) if change_action not in (Change.ACTION_UPLOAD, Change.ACTION_AUTO): # Update translation stats self.translation.invalidate_cache() # Update user stats user.profile.translated += 1 user.profile.save() # Update related source strings if working on a template if self.translation.is_template: self.update_source_units(self.old_unit.source, user) return True
def save_backend(self, request, propagate=True, change_action=None, user=None): """ Stores unit to backend. Optional user parameters defines authorship of a change. This should be always called in a trasaction with updated unit locked for update. """ # For case when authorship specified, use user from request if user is None or (user.is_anonymous and request): user = request.user # Commit possible previous changes on this unit if self.pending: try: change = self.change_set.content().order_by('-timestamp')[0] except IndexError as error: # This is probably bug in the change data, fallback by using # any change entry report_error(error, request) change = self.change_set.all().order_by('-timestamp')[0] if change.author_id != request.user.id: self.translation.commit_pending('pending unit', request) # Propagate to other projects # This has to be done before changing source/content_hash for template if propagate: self.propagate(request, change_action) # Return if there was no change # We have to explicitly check for fuzzy flag change on monolingual # files, where we handle it ourselves without storing to backend if (self.old_unit.state == self.state and self.old_unit.target == self.target): return False if self.translation.is_template: self.source = self.target self.content_hash = calculate_hash(self.source, self.context) # Unit is pending for write self.pending = True # Update translated flag (not fuzzy and at least one translation) translation = bool(max(self.get_target_plurals())) if self.state >= STATE_TRANSLATED and not translation: self.state = STATE_EMPTY elif self.state == STATE_EMPTY and translation: self.state = STATE_TRANSLATED # Save updated unit to database self.save() # Run source checks self.source_info.run_checks(unit=self) # Generate Change object for this change self.generate_change(request, user, change_action) if change_action not in (Change.ACTION_UPLOAD, Change.ACTION_AUTO): # Update translation stats self.translation.invalidate_cache() # Update user stats user.profile.translated += 1 user.profile.save() # Notify subscribed users about new translation from weblate.accounts.notifications import notify_new_translation notify_new_translation(self, self.old_unit, user) # Update related source strings if working on a template if self.translation.is_template: self.update_source_units(self.old_unit.source, user) return True
def save_backend(self, request, propagate=True, change_action=None, user=None): """ Stores unit to backend. Optional user parameters defines authorship of a change. This should be always called in a trasaction with updated unit locked for update. """ # For case when authorship specified, use user from request if user is None or (user.is_anonymous and request): user = request.user # Commit possible previous changes on this unit if self.pending: change_author = self.get_last_content_change(request)[0] if change_author.id != user.id: self.translation.commit_pending('pending unit', request) # Propagate to other projects # This has to be done before changing source/content_hash for template propagated = False if propagate: propagated = self.propagate(request, change_action) # Return if there was no change # We have to explicitly check for fuzzy flag change on monolingual # files, where we handle it ourselves without storing to backend if self.old_unit.state == self.state and self.old_unit.target == self.target: return False if self.translation.is_template: self.source = self.target self.content_hash = calculate_hash(self.source, self.context) # Unit is pending for write self.pending = True # Update translated flag (not fuzzy and at least one translation) translation = bool(max(self.get_target_plurals())) if self.state >= STATE_TRANSLATED and not translation: self.state = STATE_EMPTY elif self.state == STATE_EMPTY and translation: self.state = STATE_TRANSLATED # Save updated unit to database self.save() # Run source checks self.source_info.run_checks(unit=self) # Generate Change object for this change self.generate_change(request, user, change_action) if change_action not in (Change.ACTION_UPLOAD, Change.ACTION_AUTO): # Update translation stats self.translation.invalidate_cache(recurse=propagated) # Update user stats user.profile.translated += 1 user.profile.save() # Update related source strings if working on a template if self.translation.is_template: self.update_source_units(self.old_unit.source, user) return True
def save_backend(self, request, propagate=True, gen_change=True, change_action=None, user=None): """ Stores unit to backend. Optional user parameters defines authorship of a change. This should be always called in a trasaction with updated unit locked for update. """ # For case when authorship specified, use user from request if user is None or user.is_anonymous: user = request.user # Commit possible previous changes on this unit if self.pending: change = self.change_set.content().order_by('-timestamp')[0] if change.author_id != request.user.id: self.translation.commit_pending(request) # Return if there was no change # We have to explicitly check for fuzzy flag change on monolingual # files, where we handle it ourselves without storing to backend if (self.old_unit.state == self.state and self.old_unit.target == self.target): # Propagate if we should if propagate: self.propagate(request, change_action) return False # Propagate to other projects # This has to be done before changing source/content_hash for template if propagate: self.propagate(request, change_action) if self.translation.is_template: self.source = self.target self.content_hash = calculate_hash(self.source, self.context) # Unit is pending for write self.pending = True # Update translated flag (not fuzzy and at least one translation) translation = bool(max(self.get_target_plurals())) if self.state == STATE_TRANSLATED and not translation: self.state = STATE_EMPTY elif self.state == STATE_EMPTY and translation: self.state = STATE_TRANSLATED # Save updated unit to database self.save(backend=True) old_translated = self.translation.stats.translated if change_action not in (Change.ACTION_UPLOAD, Change.ACTION_AUTO): # Update translation stats self.translation.invalidate_cache() # Update user stats user.profile.translated += 1 user.profile.save() # Notify subscribed users about new translation notify_new_translation(self, self.old_unit, user) # Generate Change object for this change if gen_change: self.generate_change(request, user, change_action) # Force commiting on completing translation translated = self.translation.stats.translated if (old_translated < translated and translated == self.translation.stats.all): Change.objects.create(translation=self.translation, action=Change.ACTION_COMPLETE, user=user, author=user) self.translation.commit_pending(request) # Update related source strings if working on a template if self.translation.is_template: self.update_source_units(self.old_unit.source, user) return True
def save_backend(self, request, propagate=True, gen_change=True, change_action=None, user=None): """ Stores unit to backend. Optional user parameters defines authorship of a change. This should be always called in a trasaction with updated unit locked for update. """ # For case when authorship specified, use user from request if user is None or (user.is_anonymous and request): user = request.user # Commit possible previous changes on this unit if self.pending: change = self.change_set.content().order_by('-timestamp')[0] if change.author_id != request.user.id: self.translation.commit_pending(request) # Return if there was no change # We have to explicitly check for fuzzy flag change on monolingual # files, where we handle it ourselves without storing to backend if (self.old_unit.state == self.state and self.old_unit.target == self.target): # Propagate if we should if propagate: self.propagate(request, change_action) return False # Propagate to other projects # This has to be done before changing source/content_hash for template if propagate: self.propagate(request, change_action) if self.translation.is_template: self.source = self.target self.content_hash = calculate_hash(self.source, self.context) # Unit is pending for write self.pending = True # Update translated flag (not fuzzy and at least one translation) translation = bool(max(self.get_target_plurals())) if self.state == STATE_TRANSLATED and not translation: self.state = STATE_EMPTY elif self.state == STATE_EMPTY and translation: self.state = STATE_TRANSLATED # Save updated unit to database self.save(backend=True) old_translated = self.translation.stats.translated if change_action not in (Change.ACTION_UPLOAD, Change.ACTION_AUTO): # Update translation stats self.translation.invalidate_cache() # Update user stats user.profile.translated += 1 user.profile.save() # Notify subscribed users about new translation from weblate.accounts.notifications import notify_new_translation notify_new_translation(self, self.old_unit, user) # Generate Change object for this change if gen_change: self.generate_change(request, user, change_action) # Force commiting on completing translation translated = self.translation.stats.translated if (old_translated < translated and translated == self.translation.stats.all): Change.objects.create( translation=self.translation, action=Change.ACTION_COMPLETE, user=user, author=user ) self.translation.commit_pending(request) # Update related source strings if working on a template if self.translation.is_template: self.update_source_units(self.old_unit.source, user) return True
def save_backend(self, request, propagate=True, gen_change=True, change_action=None, user=None): """ Stores unit to backend. Optional user parameters defines authorship of a change. """ # For case when authorship specified, use user from request if user is None or user.is_anonymous: user = request.user # Update lock timestamp self.update_lock(request, user, change_action) # Store to backend try: (saved, pounit) = self.translation.update_unit(self, request, user) except FileLockException: self.log_error('failed to lock backend for %s!', self) messages.error( request, _( 'Failed to store message in the backend, ' 'lock timeout occurred!' ) ) return False # Handle situation when backend did not find the message if pounit is None: self.log_error('message %s disappeared!', self) messages.error( request, _( 'Message not found in backend storage, ' 'it is probably corrupted.' ) ) # Try reloading from backend self.translation.check_sync(True) return False # Return if there was no change # We have to explicitly check for fuzzy flag change on monolingual # files, where we handle it ourselves without storing to backend if (not saved and self.old_unit.fuzzy == self.fuzzy and self.old_unit.target == self.target): # Propagate if we should if propagate: self.propagate(request, change_action) return False # Update translated flag self.translated = pounit.is_translated() # Update comments as they might have been changed (eg, fuzzy flag # removed) self.flags = pounit.get_flags() if self.translation.is_template(): self.source = self.target self.content_hash = calculate_hash(self.source, self.context) # Save updated unit to database self.save(backend=True) # Update translation stats old_translated = self.translation.translated if change_action != Change.ACTION_UPLOAD: self.translation.update_stats() # Notify subscribed users about new translation notify_new_translation(self, self.old_unit, user) # Update user stats user.profile.translated += 1 user.profile.save() # Generate Change object for this change if gen_change: self.generate_change(request, user, self.old_unit, change_action) # Force commiting on completing translation if (old_translated < self.translation.translated and self.translation.translated == self.translation.total): self.translation.commit_pending(request) Change.objects.create( translation=self.translation, action=Change.ACTION_COMPLETE, user=user, author=user ) # Update related source strings if working on a template if self.translation.is_template(): self.update_source_units(self.old_unit.source) # Propagate to other projects if propagate: self.propagate(request, change_action) return True
def test_hash_unicode(self): """Ensure hash works for unicode""" text = 'Příšerně žluťoučký kůň úpěl ďábelské ódy' text_hash = calculate_hash(None, text) self.assertEqual(text_hash, calculate_hash(None, text))
def test_hash(self): """Ensure hash is not changing.""" text = "Message" text_hash = calculate_hash(text) self.assertEqual(text_hash, 8445691827737211251) self.assertEqual(text_hash, calculate_hash(text))
def test_checksum(self): """Hash to checksum conversion.""" text_hash = calculate_hash("Message") checksum = hash_to_checksum(text_hash) self.assertEqual(checksum, "f5351ff85ab23173") self.assertEqual(text_hash, checksum_to_hash(checksum))
def save_backend(self, user, propagate=True, change_action=None, author=None): """Stores unit to backend. Optional user parameters defines authorship of a change. This should be always called in a transaction with updated unit locked for update. """ # For case when authorship specified, use user author = author or user # Commit possible previous changes on this unit if self.pending: change_author = self.get_last_content_change()[0] if change_author != author: self.translation.commit_pending("pending unit", user, force=True) # Propagate to other projects # This has to be done before changing source/content_hash for template if propagate: self.propagate(user, change_action, author=author) # Return if there was no change # We have to explicitly check for fuzzy flag change on monolingual # files, where we handle it ourselves without storing to backend if self.old_unit.state == self.state and self.old_unit.target == self.target: return False if self.translation.is_source and not self.translation.component.intermediate: self.source = self.target self.content_hash = calculate_hash(self.source, self.context) # Unit is pending for write self.pending = True # Update translated flag (not fuzzy and at least one translation) translation = bool(max(self.get_target_plurals())) if self.state >= STATE_TRANSLATED and not translation: self.state = STATE_EMPTY elif self.state == STATE_EMPTY and translation: self.state = STATE_TRANSLATED self.original_state = self.state # Save updated unit to database self.save() # Generate Change object for this change self.generate_change(user or author, author, change_action) if change_action not in ( Change.ACTION_UPLOAD, Change.ACTION_AUTO, Change.ACTION_BULK_EDIT, ): # Update translation stats self.translation.invalidate_cache() # Update user stats author.profile.increase_count("translated") # Update related source strings if working on a template if self.translation.is_template and self.old_unit.target != self.target: self.update_source_units(self.old_unit.target, user or author, author) return True
def save_backend(self, request, propagate=True, gen_change=True, change_action=None, user=None): """ Stores unit to backend. Optional user parameters defines authorship of a change. """ # For case when authorship specified, use user from request if user is None: user = request.user # Update lock timestamp self.update_lock(request, user, change_action) # Store to backend try: (saved, pounit) = self.translation.update_unit(self, request, user) except FileLockException: self.log_error('failed to lock backend for %s!', self) messages.error( request, _('Failed to store message in the backend, ' 'lock timeout occurred!')) return False # Handle situation when backend did not find the message if pounit is None: self.log_error('message %s disappeared!', self) messages.error( request, _('Message not found in backend storage, ' 'it is probably corrupted.')) # Try reloading from backend self.translation.check_sync(True) return False # Return if there was no change # We have to explicitly check for fuzzy flag change on monolingual # files, where we handle it ourselves without storing to backend if (not saved and self.old_unit.fuzzy == self.fuzzy and self.old_unit.target == self.target): # Propagate if we should if propagate: self.propagate(request, change_action) return False # Update translated flag self.translated = pounit.is_translated() # Update comments as they might have been changed (eg, fuzzy flag # removed) self.flags = pounit.get_flags() if self.translation.is_template(): self.source = self.target self.content_hash = calculate_hash(self.source, self.context) # Save updated unit to database self.save(backend=True) # Update translation stats old_translated = self.translation.translated if change_action != Change.ACTION_UPLOAD: self.translation.update_stats() # Notify subscribed users about new translation notify_new_translation(self, self.old_unit, user) # Update user stats user.profile.translated += 1 user.profile.save() # Generate Change object for this change if gen_change: self.generate_change(request, user, self.old_unit, change_action) # Force commiting on completing translation if (old_translated < self.translation.translated and self.translation.translated == self.translation.total): self.translation.commit_pending(request) Change.objects.create(translation=self.translation, action=Change.ACTION_COMPLETE, user=user, author=user) # Update related source strings if working on a template if self.translation.is_template(): self.update_source_units(self.old_unit.source) # Propagate to other projects if propagate: self.propagate(request, change_action) return True
def update_from_unit(self, unit, pos, created): """Update Unit from ttkit unit.""" component = self.translation.component self.is_batch_update = True # Get unit attributes try: location = unit.locations flags = unit.flags target = unit.target self.check_valid(split_plural(target)) source = unit.source self.check_valid(split_plural(source)) context = unit.context self.check_valid([context]) note = unit.notes previous_source = unit.previous_source content_hash = unit.content_hash except Exception as error: self.translation.component.handle_parse_error( error, self.translation) # Ensure we track source string for bilingual if not self.translation.is_source: source_info = component.get_source( self.id_hash, source=source, target=source, context=context, content_hash=calculate_hash(source, context), position=0, location=location, flags=flags, ) self.extra_context = source_info.extra_context self.extra_flags = source_info.extra_flags self.__dict__["source_info"] = source_info # Calculate state state = self.get_unit_state(unit, flags) self.original_state = self.get_unit_state(unit, None) # Monolingual files handling (without target change) if not created and unit.template is not None and target == self.target: if source != self.source and state >= STATE_TRANSLATED: if self.previous_source == self.source and self.fuzzy: # Source change was reverted previous_source = '' state = STATE_TRANSLATED else: # Store previous source and fuzzy flag for monolingual if previous_source == '': previous_source = self.source state = STATE_FUZZY elif self.state in (STATE_FUZZY, STATE_APPROVED): # We should keep calculated flags if translation was # not changed outside previous_source = self.previous_source state = self.state # Update checks on fuzzy update or on content change same_target = target == self.target same_source = source == self.source and context == self.context same_state = state == self.state and flags == self.flags # Check if we actually need to change anything # pylint: disable=too-many-boolean-expressions if (location == self.location and flags == self.flags and same_source and same_target and same_state and note == self.note and pos == self.position and content_hash == self.content_hash and previous_source == self.previous_source): return # Store updated values self.position = pos self.location = location self.flags = flags self.source = source self.target = target self.state = state self.context = context self.note = note self.content_hash = content_hash self.previous_source = previous_source self.update_priority(save=False) # Sanitize number of plurals if self.is_plural(): self.target = join_plural(self.get_target_plurals()) if created: unit_pre_create.send(sender=self.__class__, unit=self) # Save into database self.save( force_insert=created, same_content=same_source and same_target, same_state=same_state, ) # Track updated sources for source checks if self.translation.is_template: component.updated_sources[self.id_hash] = self # Update unit labels if not self.translation.is_source: self.labels.set(self.source_info.labels.all())
def translate_cache_key(self, source, language, text): if not self.cache_translations: return None return "mt:{}:{}:{}".format(self.mtid, calculate_hash(source, language), calculate_hash(None, text))
def update_from_unit(self, unit, pos, created): """Update Unit from ttkit unit.""" component = self.translation.component self.is_batch_update = True # Get unit attributes try: location = unit.locations flags = unit.flags target = unit.target self.check_valid(split_plural(target)) source = unit.source self.check_valid(split_plural(source)) context = unit.context self.check_valid([context]) note = unit.notes previous_source = unit.previous_source content_hash = unit.content_hash except Exception as error: report_error(cause="Unit update error") self.translation.component.handle_parse_error( error, self.translation) # Ensure we track source string for bilingual if not self.translation.is_source: source_info = component.get_source( self.id_hash, create={ "source": source, "target": source, "context": context, "content_hash": calculate_hash(source, context), "position": pos, "note": note, "location": location, "flags": flags, }, ) if (not component.has_template() and not source_info.source_updated and (pos != source_info.position or location != source_info.location or flags != source_info.flags or note != source_info.note)): source_info.position = pos source_info.source_updated = True source_info.location = location source_info.flags = flags source_info.note = note source_info.save( update_fields=["position", "location", "flags", "note"], same_content=True, same_state=True, ) self.explanation = source_info.explanation self.extra_flags = source_info.extra_flags self.__dict__["source_info"] = source_info # Calculate state state = self.get_unit_state(unit, flags) self.original_state = self.get_unit_state(unit, None) # Has source changed same_source = source == self.source and context == self.context # Monolingual files handling (without target change) if (not created and state != STATE_READONLY and unit.template is not None and target == self.target): if not same_source and state in (STATE_TRANSLATED, STATE_APPROVED): if self.previous_source == self.source and self.fuzzy: # Source change was reverted previous_source = "" state = STATE_TRANSLATED else: # Store previous source and fuzzy flag for monolingual if previous_source == "": previous_source = self.source state = STATE_FUZZY elif self.state in (STATE_FUZZY, STATE_APPROVED): # We should keep calculated flags if translation was # not changed outside previous_source = self.previous_source state = self.state # Update checks on fuzzy update or on content change same_target = target == self.target same_state = state == self.state and flags == self.flags # Check if we actually need to change anything # pylint: disable=too-many-boolean-expressions if (location == self.location and flags == self.flags and same_source and same_target and same_state and note == self.note and pos == self.position and content_hash == self.content_hash and previous_source == self.previous_source): return # Store updated values self.position = pos self.location = location self.flags = flags self.source = source self.target = target self.state = state self.context = context self.note = note self.content_hash = content_hash self.previous_source = previous_source self.update_priority(save=False) # Sanitize number of plurals if self.is_plural(): self.target = join_plural(self.get_target_plurals()) if created: unit_pre_create.send(sender=self.__class__, unit=self) # Save into database self.save( force_insert=created, same_content=same_source and same_target, same_state=same_state, ) # Track updated sources for source checks if self.translation.is_template: component.updated_sources[self.id_hash] = self # Update unit labels if not self.translation.is_source: self.labels.set(self.source_info.labels.all()) # Indicate source string change if not same_source and previous_source: Change.objects.create( unit=self, action=Change.ACTION_SOURCE_CHANGE, old=previous_source, target=self.source, )
def translate(self, language, text, unit, user): """Return list of machine translations.""" if text == '': return [] if self.is_rate_limited(): return [] language = self.convert_language(language) source = self.convert_language( unit.translation.component.project.source_language.code ) if not self.is_supported(source, language): # Try without country code if '_' in language or '-' in language: language = language.replace('-', '_').split('_')[0] if source == language: return [] if not self.is_supported(source, language): return [] else: return [] cache_key = None if self.cache_translations: cache_key = 'mt:{}:{}:{}'.format( self.mtid, calculate_hash(source, language), hash_to_checksum(calculate_hash(None, text)), ) result = cache.get(cache_key) if result is not None: return result try: translations = self.download_translations( source, language, text, unit, user ) result = [ { 'text': trans[0], 'quality': trans[1], 'service': trans[2], 'source': trans[3] } for trans in translations ] if cache_key: cache.set(cache_key, result, 7 * 86400) return result except Exception as exc: if self.is_rate_limit_error(exc): self.set_rate_limit() self.report_error( exc, 'Failed to fetch translations from %s', ) raise MachineTranslationError('{0}: {1}'.format( exc.__class__.__name__, str(exc) ))
def add_unit( # noqa: C901 self, request, context: str, source: Union[str, List[str]], target: Optional[Union[str, List[str]]] = None, extra_flags: str = "", explanation: str = "", auto_context: bool = False, is_batch_update: bool = False, ): user = request.user if request else None component = self.component if self.is_source: translations = [self] translations.extend(component.translation_set.exclude(id=self.id)) else: translations = [component.source_translation, self] has_template = component.has_template() source_unit = None result = None # Automatic context suffix = 0 base = context while self.unit_set.filter(context=context, source=source).exists(): suffix += 1 context = f"{base}{suffix}" for translation in translations: is_source = translation.is_source kwargs = {} if has_template: kwargs["pending"] = is_source else: kwargs["pending"] = not is_source if kwargs["pending"]: kwargs["details"] = {"add_unit": True} if is_source: current_target = source kwargs["extra_flags"] = extra_flags kwargs["explanation"] = explanation else: current_target = target if current_target is None: current_target = "" if isinstance(current_target, list): current_target = join_plural(current_target) if isinstance(source, list): source = join_plural(source) if has_template: id_hash = calculate_hash(context) else: id_hash = calculate_hash(source, context) # When adding to a target the source string can already exist unit = None if not self.is_source and is_source: try: unit = translation.unit_set.get(id_hash=id_hash) flags = Flags(unit.extra_flags) flags.merge(extra_flags) new_flags = flags.format() if unit.extra_flags != new_flags or unit.explanation != explanation: unit.extra_flags = new_flags unit.explanation = explanation unit.save( update_fields=["extra_flags", "explanation"], same_content=True, ) except Unit.DoesNotExist: pass if unit is None: unit = Unit( translation=translation, context=context, source=source, target=current_target, state=STATE_TRANSLATED if bool(current_target) else STATE_EMPTY, source_unit=source_unit, id_hash=id_hash, position=0, **kwargs, ) unit.is_batch_update = is_batch_update unit.save(force_insert=True) Change.objects.create( unit=unit, action=Change.ACTION_NEW_UNIT, target=current_target, user=user, author=user, ) # The source language is always first in the translations array if source_unit is None: source_unit = unit if translation == self: result = unit if not is_batch_update: component.update_variants() component.sync_terminology() return result