def id_hash(self):
"""Return hash of source string, used for quick lookup.
We use siphash as it is fast and works well for our purpose.
"""
if self.template is None:
return calculate_hash(self.source, self.context)
return calculate_hash(None, self.context)
def test_hash(self):
"""Ensure hash is not changing"""
text = 'Message'
text_hash = calculate_hash(None, text)
self.assertEqual(
text_hash,
calculate_hash(None, text)
)
def translate_cache_key(self, source, language, text):
if not self.cache_translations:
return None
return 'mt:{}:{}:{}'.format(
self.mtid,
calculate_hash(source, language),
calculate_hash(None, text),
)
def test_hash_unicode(self):
"""Ensure hash works for unicode"""
text = 'Příšerně žluťoučký kůň úpěl ďábelské ódy'
text_hash = calculate_hash(None, text)
self.assertEqual(
text_hash,
calculate_hash(None, text)
)
def test_hash_context(self):
"""Ensure hash works with context"""
text = 'Message'
context = 'Context'
text_hash = calculate_hash(context, text)
self.assertEqual(
text_hash,
calculate_hash(context, text)
)
def get_id_hash(self):
"""Return hash of source string, used for quick lookup.
We use siphash as it is fast and works well for our purpose.
"""
if self.id_hash is None:
if self.template is None:
self.id_hash = calculate_hash(
self.get_source(), self.get_context()
)
else:
self.id_hash = calculate_hash(
None, self.get_context()
)
return self.id_hash
def test_checksum(self):
"""Hash to checksum conversion"""
text_hash = calculate_hash(None, 'Message')
checksum = hash_to_checksum(text_hash)
self.assertEqual(
text_hash,
checksum_to_hash(checksum)
)
def calculate_id_hash(apps, schema_editor):
Unit = apps.get_model('trans', 'Unit')
Comment = apps.get_model('trans', 'Comment')
Check = apps.get_model('trans', 'Check')
Suggestion = apps.get_model('trans', 'Suggestion')
Source = apps.get_model('trans', 'Source')
content_models = (Comment, Check, Suggestion)
units = Unit.objects.prefetch_related(
'translation', 'translation__language',
'translation__subproject',
'translation__subproject__project',
)
for unit in units.iterator():
unit.content_hash = calculate_hash(unit.source, unit.context)
if unit.translation.subproject.template:
unit.id_hash = calculate_hash(None, unit.context)
else:
unit.id_hash = unit.content_hash
unit.save(update_fields=['content_hash', 'id_hash'])
for model in content_models:
model.objects.filter(
(Q(language=unit.translation.language) | Q(language=None)) &
Q(project=unit.translation.subproject.project) &
Q(contentsum=unit.contentsum)
).update(
content_hash=unit.content_hash
)
Source.objects.filter(
subproject=unit.translation.subproject,
checksum=unit.checksum
).update(
id_hash=unit.id_hash
)
# Remove stale instances
for model in content_models:
model.objects.filter(content_hash=0).delete()
Source.objects.filter(id_hash=0).delete()
def get_content_hash(self):
"""Return hash of source string and context, used for quick lookup."""
if self.template is None:
return self.get_id_hash()
if self.content_hash is None:
self.content_hash = calculate_hash(
self.get_source(), self.get_context()
)
return self.content_hash
def content_hash(self):
"""Return hash of source string and context, used for quick lookup."""
if self.template is None:
return self.id_hash
return calculate_hash(self.source, self.context)
def get_target_hash(self):
return calculate_hash(None, self.target)
def migrate_glossaries(apps, schema_editor): # noqa: C901
Project = apps.get_model("trans", "Project")
Language = apps.get_model("lang", "Language")
db_alias = schema_editor.connection.alias
projects = Project.objects.using(db_alias).all()
total = len(projects)
processed = 0
for processed, project in enumerate(projects):
component_slugs = set(project.component_set.values_list("slug", flat=True))
percent = int(100 * processed / total)
print(f"Migrating glossaries {percent}% [{processed}/{total}]...{project.name}")
glossaries = project.glossary_set.all()
try:
license = project.component_set.exclude(license="").values_list(
"license", flat=True
)[0]
except IndexError:
license = ""
for glossary in glossaries:
if len(glossaries) == 1:
name = "Glossary"
slug = "glossary"
else:
name = f"Glossary: {glossary.name}"
slug = f"glossary-{slugify(glossary.name)}"
base_name = name
base_slug = slug
# Create component
attempts = 0
while True:
if slug not in component_slugs:
component = create_glossary(project, name, slug, glossary, license)
component_slugs.add(slug)
break
attempts += 1
name = f"{base_name} - {attempts}"
slug = f"{base_slug}-{attempts}"
repo_path = os.path.join(settings.DATA_DIR, "vcs", project.slug, slug)
# Create VCS repository
repo = LocalRepository.from_files(repo_path, {})
# Migrate links
component.links.set(glossary.links.all())
# Create source translation
source_translation = component.translation_set.create(
language=glossary.source_language,
check_flags="read-only",
filename="",
plural=glossary.source_language.plural_set.filter(source=0)[0],
language_code=glossary.source_language.code,
)
source_units = {}
# Get list of languages
languages = (
Language.objects.filter(term__glossary=glossary)
.exclude(pk=glossary.source_language.pk)
.distinct()
)
# Migrate ters
for language in languages:
base_filename = f"{language.code}.tbx"
filename = os.path.join(repo_path, base_filename)
# Create translation object
translation = component.translation_set.create(
language=language,
plural=language.plural_set.filter(source=0)[0],
filename=base_filename,
language_code=language.code,
)
# Create store file
TBXFormat.create_new_file(filename, language.code, "")
store = TBXFormat(
filename,
language_code=language.code,
source_language=glossary.source_language.code,
)
id_hashes = set()
for position, term in enumerate(
glossary.term_set.filter(language=language)
):
source = valid_chars_only(term.source)
target = valid_chars_only(term.target)
context = ""
# Store to the file
id_hash = calculate_hash(source, context)
offset = 0
while id_hash in id_hashes:
offset += 1
context = str(offset)
id_hash = calculate_hash(source, context)
id_hashes.add(id_hash)
if id_hash not in source_units:
source_units[id_hash] = source_translation.unit_set.create(
context=context,
source=source,
target=source,
state=STATE_READONLY,
position=position,
num_words=len(source.split()),
id_hash=id_hash,
)
source_units[id_hash].source_unit = source_units[id_hash]
source_units[id_hash].save()
store.new_unit(context, source, target)
# Migrate database
unit = translation.unit_set.create(
context=context,
source=source,
target=target,
state=STATE_TRANSLATED,
position=position,
num_words=len(source.split()),
id_hash=id_hash,
source_unit=source_units[id_hash],
)
# Adjust history entries to include unit details,
# language and project should be already set
term.change_set.update(
unit=unit,
translation=translation,
component=component,
)
store.save()
# Update translation hash
translation.revision = repo.get_object_hash(filename)
translation.save(update_fields=["revision"])
# Commit files
with repo.lock:
repo.execute(["add", repo_path])
if repo.needs_commit():
repo.commit("Migrate glossary content")
if total:
print(f"Migrating glossaries completed [{total}/{total}]")
def save_backend(self,
request,
propagate=True,
gen_change=True,
change_action=None,
user=None):
"""
Stores unit to backend.
Optional user parameters defines authorship of a change.
"""
# For case when authorship specified, use user from request
if user is None or user.is_anonymous:
user = request.user
# Update lock timestamp
self.update_lock(request, user, change_action)
# Commit possible previous changes by other author
self.translation.commit_pending(request, get_author_name(user))
# Return if there was no change
# We have to explicitly check for fuzzy flag change on monolingual
# files, where we handle it ourselves without storing to backend
if (self.old_unit.fuzzy == self.fuzzy
and self.old_unit.target == self.target):
# Propagate if we should
if propagate:
self.propagate(request, change_action)
return False
# Propagate to other projects
# This has to be done before changing source/content_hash for template
if propagate:
self.propagate(request, change_action)
if self.translation.is_template():
self.source = self.target
self.content_hash = calculate_hash(self.source, self.context)
# Unit is pending for write
self.pending = True
# Update translated flag (not fuzzy and at least one transltion)
self.translated = (not self.fuzzy
and bool(max(self.get_target_plurals())))
# Save updated unit to database
self.save(backend=True)
# Update translation stats
old_translated = self.translation.translated
if change_action != Change.ACTION_UPLOAD:
self.translation.update_stats()
self.translation.store_hash()
# Notify subscribed users about new translation
notify_new_translation(self, self.old_unit, user)
# Update user stats
user.profile.translated += 1
user.profile.save()
# Generate Change object for this change
if gen_change:
self.generate_change(request, user, self.old_unit, change_action)
# Force commiting on completing translation
if (old_translated < self.translation.translated
and self.translation.translated == self.translation.total):
self.translation.commit_pending(request)
Change.objects.create(translation=self.translation,
action=Change.ACTION_COMPLETE,
user=user,
author=user)
# Update related source strings if working on a template
if self.translation.is_template():
self.update_source_units(self.old_unit.source)
return True
def test_hash_unicode(self):
"""Ensure hash works for unicode."""
text = "Příšerně žluťoučký kůň úpěl ďábelské ódy"
text_hash = calculate_hash(text)
self.assertEqual(text_hash, -4296353750398394478)
self.assertEqual(text_hash, calculate_hash(text))
def test_hash(self):
"""Ensure hash is not changing"""
text = 'Message'
text_hash = calculate_hash(None, text)
self.assertEqual(text_hash, calculate_hash(None, text))
def content_hash(self):
"""Return hash of source string and context, used for quick lookup."""
if self.template is None:
return self.id_hash
return calculate_hash(self.source, self.context)
def test_checksum(self):
"""Hash to checksum conversion"""
text_hash = calculate_hash(None, 'Message')
checksum = hash_to_checksum(text_hash)
self.assertEqual(text_hash, checksum_to_hash(checksum))
def test_hash_context(self):
"""Ensure hash works with context"""
text = 'Message'
context = 'Context'
text_hash = calculate_hash(context, text)
self.assertEqual(text_hash, calculate_hash(context, text))
def content_hash(self):
if self.translation.component.template:
return calculate_hash(self.source, self.context)
return self.id_hash
def save_backend(self, request, propagate=True, change_action=None,
user=None):
"""
Stores unit to backend.
Optional user parameters defines authorship of a change.
This should be always called in a trasaction with updated unit
locked for update.
"""
# For case when authorship specified, use user from request
if user is None or (user.is_anonymous and request):
user = request.user
# Commit possible previous changes on this unit
if self.pending:
change_author = self.get_last_content_change(request)[0]
if change_author.id != user.id:
self.translation.commit_pending('pending unit', request)
# Propagate to other projects
# This has to be done before changing source/content_hash for template
if propagate:
self.propagate(request, change_action)
# Return if there was no change
# We have to explicitly check for fuzzy flag change on monolingual
# files, where we handle it ourselves without storing to backend
if (self.old_unit.state == self.state and
self.old_unit.target == self.target):
return False
if self.translation.is_template:
self.source = self.target
self.content_hash = calculate_hash(self.source, self.context)
# Unit is pending for write
self.pending = True
# Update translated flag (not fuzzy and at least one translation)
translation = bool(max(self.get_target_plurals()))
if self.state >= STATE_TRANSLATED and not translation:
self.state = STATE_EMPTY
elif self.state == STATE_EMPTY and translation:
self.state = STATE_TRANSLATED
# Save updated unit to database
self.save()
# Run source checks
self.source_info.run_checks(unit=self)
# Generate Change object for this change
self.generate_change(request, user, change_action)
if change_action not in (Change.ACTION_UPLOAD, Change.ACTION_AUTO):
# Update translation stats
self.translation.invalidate_cache()
# Update user stats
user.profile.translated += 1
user.profile.save()
# Update related source strings if working on a template
if self.translation.is_template:
self.update_source_units(self.old_unit.source, user)
return True
def save_backend(self,
request,
propagate=True,
change_action=None,
user=None):
"""
Stores unit to backend.
Optional user parameters defines authorship of a change.
This should be always called in a trasaction with updated unit
locked for update.
"""
# For case when authorship specified, use user from request
if user is None or (user.is_anonymous and request):
user = request.user
# Commit possible previous changes on this unit
if self.pending:
try:
change = self.change_set.content().order_by('-timestamp')[0]
except IndexError as error:
# This is probably bug in the change data, fallback by using
# any change entry
report_error(error, request)
change = self.change_set.all().order_by('-timestamp')[0]
if change.author_id != request.user.id:
self.translation.commit_pending('pending unit', request)
# Propagate to other projects
# This has to be done before changing source/content_hash for template
if propagate:
self.propagate(request, change_action)
# Return if there was no change
# We have to explicitly check for fuzzy flag change on monolingual
# files, where we handle it ourselves without storing to backend
if (self.old_unit.state == self.state
and self.old_unit.target == self.target):
return False
if self.translation.is_template:
self.source = self.target
self.content_hash = calculate_hash(self.source, self.context)
# Unit is pending for write
self.pending = True
# Update translated flag (not fuzzy and at least one translation)
translation = bool(max(self.get_target_plurals()))
if self.state >= STATE_TRANSLATED and not translation:
self.state = STATE_EMPTY
elif self.state == STATE_EMPTY and translation:
self.state = STATE_TRANSLATED
# Save updated unit to database
self.save()
# Run source checks
self.source_info.run_checks(unit=self)
# Generate Change object for this change
self.generate_change(request, user, change_action)
if change_action not in (Change.ACTION_UPLOAD, Change.ACTION_AUTO):
# Update translation stats
self.translation.invalidate_cache()
# Update user stats
user.profile.translated += 1
user.profile.save()
# Notify subscribed users about new translation
from weblate.accounts.notifications import notify_new_translation
notify_new_translation(self, self.old_unit, user)
# Update related source strings if working on a template
if self.translation.is_template:
self.update_source_units(self.old_unit.source, user)
return True
def save_backend(self, request, propagate=True, change_action=None,
user=None):
"""
Stores unit to backend.
Optional user parameters defines authorship of a change.
This should be always called in a trasaction with updated unit
locked for update.
"""
# For case when authorship specified, use user from request
if user is None or (user.is_anonymous and request):
user = request.user
# Commit possible previous changes on this unit
if self.pending:
change_author = self.get_last_content_change(request)[0]
if change_author.id != user.id:
self.translation.commit_pending('pending unit', request)
# Propagate to other projects
# This has to be done before changing source/content_hash for template
propagated = False
if propagate:
propagated = self.propagate(request, change_action)
# Return if there was no change
# We have to explicitly check for fuzzy flag change on monolingual
# files, where we handle it ourselves without storing to backend
if self.old_unit.state == self.state and self.old_unit.target == self.target:
return False
if self.translation.is_template:
self.source = self.target
self.content_hash = calculate_hash(self.source, self.context)
# Unit is pending for write
self.pending = True
# Update translated flag (not fuzzy and at least one translation)
translation = bool(max(self.get_target_plurals()))
if self.state >= STATE_TRANSLATED and not translation:
self.state = STATE_EMPTY
elif self.state == STATE_EMPTY and translation:
self.state = STATE_TRANSLATED
# Save updated unit to database
self.save()
# Run source checks
self.source_info.run_checks(unit=self)
# Generate Change object for this change
self.generate_change(request, user, change_action)
if change_action not in (Change.ACTION_UPLOAD, Change.ACTION_AUTO):
# Update translation stats
self.translation.invalidate_cache(recurse=propagated)
# Update user stats
user.profile.translated += 1
user.profile.save()
# Update related source strings if working on a template
if self.translation.is_template:
self.update_source_units(self.old_unit.source, user)
return True
def save_backend(self,
request,
propagate=True,
gen_change=True,
change_action=None,
user=None):
"""
Stores unit to backend.
Optional user parameters defines authorship of a change.
This should be always called in a trasaction with updated unit
locked for update.
"""
# For case when authorship specified, use user from request
if user is None or user.is_anonymous:
user = request.user
# Commit possible previous changes on this unit
if self.pending:
change = self.change_set.content().order_by('-timestamp')[0]
if change.author_id != request.user.id:
self.translation.commit_pending(request)
# Return if there was no change
# We have to explicitly check for fuzzy flag change on monolingual
# files, where we handle it ourselves without storing to backend
if (self.old_unit.state == self.state
and self.old_unit.target == self.target):
# Propagate if we should
if propagate:
self.propagate(request, change_action)
return False
# Propagate to other projects
# This has to be done before changing source/content_hash for template
if propagate:
self.propagate(request, change_action)
if self.translation.is_template:
self.source = self.target
self.content_hash = calculate_hash(self.source, self.context)
# Unit is pending for write
self.pending = True
# Update translated flag (not fuzzy and at least one translation)
translation = bool(max(self.get_target_plurals()))
if self.state == STATE_TRANSLATED and not translation:
self.state = STATE_EMPTY
elif self.state == STATE_EMPTY and translation:
self.state = STATE_TRANSLATED
# Save updated unit to database
self.save(backend=True)
old_translated = self.translation.stats.translated
if change_action not in (Change.ACTION_UPLOAD, Change.ACTION_AUTO):
# Update translation stats
self.translation.invalidate_cache()
# Update user stats
user.profile.translated += 1
user.profile.save()
# Notify subscribed users about new translation
notify_new_translation(self, self.old_unit, user)
# Generate Change object for this change
if gen_change:
self.generate_change(request, user, change_action)
# Force commiting on completing translation
translated = self.translation.stats.translated
if (old_translated < translated
and translated == self.translation.stats.all):
Change.objects.create(translation=self.translation,
action=Change.ACTION_COMPLETE,
user=user,
author=user)
self.translation.commit_pending(request)
# Update related source strings if working on a template
if self.translation.is_template:
self.update_source_units(self.old_unit.source, user)
return True
def save_backend(self, request, propagate=True, gen_change=True,
change_action=None, user=None):
"""
Stores unit to backend.
Optional user parameters defines authorship of a change.
This should be always called in a trasaction with updated unit
locked for update.
"""
# For case when authorship specified, use user from request
if user is None or (user.is_anonymous and request):
user = request.user
# Commit possible previous changes on this unit
if self.pending:
change = self.change_set.content().order_by('-timestamp')[0]
if change.author_id != request.user.id:
self.translation.commit_pending(request)
# Return if there was no change
# We have to explicitly check for fuzzy flag change on monolingual
# files, where we handle it ourselves without storing to backend
if (self.old_unit.state == self.state and
self.old_unit.target == self.target):
# Propagate if we should
if propagate:
self.propagate(request, change_action)
return False
# Propagate to other projects
# This has to be done before changing source/content_hash for template
if propagate:
self.propagate(request, change_action)
if self.translation.is_template:
self.source = self.target
self.content_hash = calculate_hash(self.source, self.context)
# Unit is pending for write
self.pending = True
# Update translated flag (not fuzzy and at least one translation)
translation = bool(max(self.get_target_plurals()))
if self.state == STATE_TRANSLATED and not translation:
self.state = STATE_EMPTY
elif self.state == STATE_EMPTY and translation:
self.state = STATE_TRANSLATED
# Save updated unit to database
self.save(backend=True)
old_translated = self.translation.stats.translated
if change_action not in (Change.ACTION_UPLOAD, Change.ACTION_AUTO):
# Update translation stats
self.translation.invalidate_cache()
# Update user stats
user.profile.translated += 1
user.profile.save()
# Notify subscribed users about new translation
from weblate.accounts.notifications import notify_new_translation
notify_new_translation(self, self.old_unit, user)
# Generate Change object for this change
if gen_change:
self.generate_change(request, user, change_action)
# Force commiting on completing translation
translated = self.translation.stats.translated
if (old_translated < translated and
translated == self.translation.stats.all):
Change.objects.create(
translation=self.translation,
action=Change.ACTION_COMPLETE,
user=user,
author=user
)
self.translation.commit_pending(request)
# Update related source strings if working on a template
if self.translation.is_template:
self.update_source_units(self.old_unit.source, user)
return True
def save_backend(self, request, propagate=True, gen_change=True,
change_action=None, user=None):
"""
Stores unit to backend.
Optional user parameters defines authorship of a change.
"""
# For case when authorship specified, use user from request
if user is None or user.is_anonymous:
user = request.user
# Update lock timestamp
self.update_lock(request, user, change_action)
# Store to backend
try:
(saved, pounit) = self.translation.update_unit(self, request, user)
except FileLockException:
self.log_error('failed to lock backend for %s!', self)
messages.error(
request,
_(
'Failed to store message in the backend, '
'lock timeout occurred!'
)
)
return False
# Handle situation when backend did not find the message
if pounit is None:
self.log_error('message %s disappeared!', self)
messages.error(
request,
_(
'Message not found in backend storage, '
'it is probably corrupted.'
)
)
# Try reloading from backend
self.translation.check_sync(True)
return False
# Return if there was no change
# We have to explicitly check for fuzzy flag change on monolingual
# files, where we handle it ourselves without storing to backend
if (not saved and
self.old_unit.fuzzy == self.fuzzy and
self.old_unit.target == self.target):
# Propagate if we should
if propagate:
self.propagate(request, change_action)
return False
# Update translated flag
self.translated = pounit.is_translated()
# Update comments as they might have been changed (eg, fuzzy flag
# removed)
self.flags = pounit.get_flags()
if self.translation.is_template():
self.source = self.target
self.content_hash = calculate_hash(self.source, self.context)
# Save updated unit to database
self.save(backend=True)
# Update translation stats
old_translated = self.translation.translated
if change_action != Change.ACTION_UPLOAD:
self.translation.update_stats()
# Notify subscribed users about new translation
notify_new_translation(self, self.old_unit, user)
# Update user stats
user.profile.translated += 1
user.profile.save()
# Generate Change object for this change
if gen_change:
self.generate_change(request, user, self.old_unit, change_action)
# Force commiting on completing translation
if (old_translated < self.translation.translated and
self.translation.translated == self.translation.total):
self.translation.commit_pending(request)
Change.objects.create(
translation=self.translation,
action=Change.ACTION_COMPLETE,
user=user,
author=user
)
# Update related source strings if working on a template
if self.translation.is_template():
self.update_source_units(self.old_unit.source)
# Propagate to other projects
if propagate:
self.propagate(request, change_action)
return True
def test_hash_unicode(self):
"""Ensure hash works for unicode"""
text = 'Příšerně žluťoučký kůň úpěl ďábelské ódy'
text_hash = calculate_hash(None, text)
self.assertEqual(text_hash, calculate_hash(None, text))
def get_target_hash(self):
return calculate_hash(None, self.target)
def test_hash(self):
"""Ensure hash is not changing."""
text = "Message"
text_hash = calculate_hash(text)
self.assertEqual(text_hash, 8445691827737211251)
self.assertEqual(text_hash, calculate_hash(text))
def test_checksum(self):
"""Hash to checksum conversion."""
text_hash = calculate_hash("Message")
checksum = hash_to_checksum(text_hash)
self.assertEqual(checksum, "f5351ff85ab23173")
self.assertEqual(text_hash, checksum_to_hash(checksum))
def save_backend(self,
user,
propagate=True,
change_action=None,
author=None):
"""Stores unit to backend.
Optional user parameters defines authorship of a change.
This should be always called in a transaction with updated unit
locked for update.
"""
# For case when authorship specified, use user
author = author or user
# Commit possible previous changes on this unit
if self.pending:
change_author = self.get_last_content_change()[0]
if change_author != author:
self.translation.commit_pending("pending unit",
user,
force=True)
# Propagate to other projects
# This has to be done before changing source/content_hash for template
if propagate:
self.propagate(user, change_action, author=author)
# Return if there was no change
# We have to explicitly check for fuzzy flag change on monolingual
# files, where we handle it ourselves without storing to backend
if self.old_unit.state == self.state and self.old_unit.target == self.target:
return False
if self.translation.is_source and not self.translation.component.intermediate:
self.source = self.target
self.content_hash = calculate_hash(self.source, self.context)
# Unit is pending for write
self.pending = True
# Update translated flag (not fuzzy and at least one translation)
translation = bool(max(self.get_target_plurals()))
if self.state >= STATE_TRANSLATED and not translation:
self.state = STATE_EMPTY
elif self.state == STATE_EMPTY and translation:
self.state = STATE_TRANSLATED
self.original_state = self.state
# Save updated unit to database
self.save()
# Generate Change object for this change
self.generate_change(user or author, author, change_action)
if change_action not in (
Change.ACTION_UPLOAD,
Change.ACTION_AUTO,
Change.ACTION_BULK_EDIT,
):
# Update translation stats
self.translation.invalidate_cache()
# Update user stats
author.profile.increase_count("translated")
# Update related source strings if working on a template
if self.translation.is_template and self.old_unit.target != self.target:
self.update_source_units(self.old_unit.target, user or author,
author)
return True
def save_backend(self,
request,
propagate=True,
gen_change=True,
change_action=None,
user=None):
"""
Stores unit to backend.
Optional user parameters defines authorship of a change.
"""
# For case when authorship specified, use user from request
if user is None:
user = request.user
# Update lock timestamp
self.update_lock(request, user, change_action)
# Store to backend
try:
(saved, pounit) = self.translation.update_unit(self, request, user)
except FileLockException:
self.log_error('failed to lock backend for %s!', self)
messages.error(
request,
_('Failed to store message in the backend, '
'lock timeout occurred!'))
return False
# Handle situation when backend did not find the message
if pounit is None:
self.log_error('message %s disappeared!', self)
messages.error(
request,
_('Message not found in backend storage, '
'it is probably corrupted.'))
# Try reloading from backend
self.translation.check_sync(True)
return False
# Return if there was no change
# We have to explicitly check for fuzzy flag change on monolingual
# files, where we handle it ourselves without storing to backend
if (not saved and self.old_unit.fuzzy == self.fuzzy
and self.old_unit.target == self.target):
# Propagate if we should
if propagate:
self.propagate(request, change_action)
return False
# Update translated flag
self.translated = pounit.is_translated()
# Update comments as they might have been changed (eg, fuzzy flag
# removed)
self.flags = pounit.get_flags()
if self.translation.is_template():
self.source = self.target
self.content_hash = calculate_hash(self.source, self.context)
# Save updated unit to database
self.save(backend=True)
# Update translation stats
old_translated = self.translation.translated
if change_action != Change.ACTION_UPLOAD:
self.translation.update_stats()
# Notify subscribed users about new translation
notify_new_translation(self, self.old_unit, user)
# Update user stats
user.profile.translated += 1
user.profile.save()
# Generate Change object for this change
if gen_change:
self.generate_change(request, user, self.old_unit, change_action)
# Force commiting on completing translation
if (old_translated < self.translation.translated
and self.translation.translated == self.translation.total):
self.translation.commit_pending(request)
Change.objects.create(translation=self.translation,
action=Change.ACTION_COMPLETE,
user=user,
author=user)
# Update related source strings if working on a template
if self.translation.is_template():
self.update_source_units(self.old_unit.source)
# Propagate to other projects
if propagate:
self.propagate(request, change_action)
return True
def update_from_unit(self, unit, pos, created):
"""Update Unit from ttkit unit."""
component = self.translation.component
self.is_batch_update = True
# Get unit attributes
try:
location = unit.locations
flags = unit.flags
target = unit.target
self.check_valid(split_plural(target))
source = unit.source
self.check_valid(split_plural(source))
context = unit.context
self.check_valid([context])
note = unit.notes
previous_source = unit.previous_source
content_hash = unit.content_hash
except Exception as error:
self.translation.component.handle_parse_error(
error, self.translation)
# Ensure we track source string for bilingual
if not self.translation.is_source:
source_info = component.get_source(
self.id_hash,
source=source,
target=source,
context=context,
content_hash=calculate_hash(source, context),
position=0,
location=location,
flags=flags,
)
self.extra_context = source_info.extra_context
self.extra_flags = source_info.extra_flags
self.__dict__["source_info"] = source_info
# Calculate state
state = self.get_unit_state(unit, flags)
self.original_state = self.get_unit_state(unit, None)
# Monolingual files handling (without target change)
if not created and unit.template is not None and target == self.target:
if source != self.source and state >= STATE_TRANSLATED:
if self.previous_source == self.source and self.fuzzy:
# Source change was reverted
previous_source = ''
state = STATE_TRANSLATED
else:
# Store previous source and fuzzy flag for monolingual
if previous_source == '':
previous_source = self.source
state = STATE_FUZZY
elif self.state in (STATE_FUZZY, STATE_APPROVED):
# We should keep calculated flags if translation was
# not changed outside
previous_source = self.previous_source
state = self.state
# Update checks on fuzzy update or on content change
same_target = target == self.target
same_source = source == self.source and context == self.context
same_state = state == self.state and flags == self.flags
# Check if we actually need to change anything
# pylint: disable=too-many-boolean-expressions
if (location == self.location and flags == self.flags and same_source
and same_target and same_state and note == self.note
and pos == self.position and content_hash == self.content_hash
and previous_source == self.previous_source):
return
# Store updated values
self.position = pos
self.location = location
self.flags = flags
self.source = source
self.target = target
self.state = state
self.context = context
self.note = note
self.content_hash = content_hash
self.previous_source = previous_source
self.update_priority(save=False)
# Sanitize number of plurals
if self.is_plural():
self.target = join_plural(self.get_target_plurals())
if created:
unit_pre_create.send(sender=self.__class__, unit=self)
# Save into database
self.save(
force_insert=created,
same_content=same_source and same_target,
same_state=same_state,
)
# Track updated sources for source checks
if self.translation.is_template:
component.updated_sources[self.id_hash] = self
# Update unit labels
if not self.translation.is_source:
self.labels.set(self.source_info.labels.all())
def translate_cache_key(self, source, language, text):
if not self.cache_translations:
return None
return "mt:{}:{}:{}".format(self.mtid,
calculate_hash(source, language),
calculate_hash(None, text))
def update_from_unit(self, unit, pos, created):
"""Update Unit from ttkit unit."""
component = self.translation.component
self.is_batch_update = True
# Get unit attributes
try:
location = unit.locations
flags = unit.flags
target = unit.target
self.check_valid(split_plural(target))
source = unit.source
self.check_valid(split_plural(source))
context = unit.context
self.check_valid([context])
note = unit.notes
previous_source = unit.previous_source
content_hash = unit.content_hash
except Exception as error:
report_error(cause="Unit update error")
self.translation.component.handle_parse_error(
error, self.translation)
# Ensure we track source string for bilingual
if not self.translation.is_source:
source_info = component.get_source(
self.id_hash,
create={
"source": source,
"target": source,
"context": context,
"content_hash": calculate_hash(source, context),
"position": pos,
"note": note,
"location": location,
"flags": flags,
},
)
if (not component.has_template() and not source_info.source_updated
and
(pos != source_info.position
or location != source_info.location
or flags != source_info.flags or note != source_info.note)):
source_info.position = pos
source_info.source_updated = True
source_info.location = location
source_info.flags = flags
source_info.note = note
source_info.save(
update_fields=["position", "location", "flags", "note"],
same_content=True,
same_state=True,
)
self.explanation = source_info.explanation
self.extra_flags = source_info.extra_flags
self.__dict__["source_info"] = source_info
# Calculate state
state = self.get_unit_state(unit, flags)
self.original_state = self.get_unit_state(unit, None)
# Has source changed
same_source = source == self.source and context == self.context
# Monolingual files handling (without target change)
if (not created and state != STATE_READONLY
and unit.template is not None and target == self.target):
if not same_source and state in (STATE_TRANSLATED, STATE_APPROVED):
if self.previous_source == self.source and self.fuzzy:
# Source change was reverted
previous_source = ""
state = STATE_TRANSLATED
else:
# Store previous source and fuzzy flag for monolingual
if previous_source == "":
previous_source = self.source
state = STATE_FUZZY
elif self.state in (STATE_FUZZY, STATE_APPROVED):
# We should keep calculated flags if translation was
# not changed outside
previous_source = self.previous_source
state = self.state
# Update checks on fuzzy update or on content change
same_target = target == self.target
same_state = state == self.state and flags == self.flags
# Check if we actually need to change anything
# pylint: disable=too-many-boolean-expressions
if (location == self.location and flags == self.flags and same_source
and same_target and same_state and note == self.note
and pos == self.position and content_hash == self.content_hash
and previous_source == self.previous_source):
return
# Store updated values
self.position = pos
self.location = location
self.flags = flags
self.source = source
self.target = target
self.state = state
self.context = context
self.note = note
self.content_hash = content_hash
self.previous_source = previous_source
self.update_priority(save=False)
# Sanitize number of plurals
if self.is_plural():
self.target = join_plural(self.get_target_plurals())
if created:
unit_pre_create.send(sender=self.__class__, unit=self)
# Save into database
self.save(
force_insert=created,
same_content=same_source and same_target,
same_state=same_state,
)
# Track updated sources for source checks
if self.translation.is_template:
component.updated_sources[self.id_hash] = self
# Update unit labels
if not self.translation.is_source:
self.labels.set(self.source_info.labels.all())
# Indicate source string change
if not same_source and previous_source:
Change.objects.create(
unit=self,
action=Change.ACTION_SOURCE_CHANGE,
old=previous_source,
target=self.source,
)
def translate(self, language, text, unit, user):
"""Return list of machine translations."""
if text == '':
return []
if self.is_rate_limited():
return []
language = self.convert_language(language)
source = self.convert_language(
unit.translation.component.project.source_language.code
)
if not self.is_supported(source, language):
# Try without country code
if '_' in language or '-' in language:
language = language.replace('-', '_').split('_')[0]
if source == language:
return []
if not self.is_supported(source, language):
return []
else:
return []
cache_key = None
if self.cache_translations:
cache_key = 'mt:{}:{}:{}'.format(
self.mtid,
calculate_hash(source, language),
hash_to_checksum(calculate_hash(None, text)),
)
result = cache.get(cache_key)
if result is not None:
return result
try:
translations = self.download_translations(
source, language, text, unit, user
)
result = [
{
'text': trans[0],
'quality': trans[1],
'service': trans[2],
'source': trans[3]
}
for trans in translations
]
if cache_key:
cache.set(cache_key, result, 7 * 86400)
return result
except Exception as exc:
if self.is_rate_limit_error(exc):
self.set_rate_limit()
self.report_error(
exc,
'Failed to fetch translations from %s',
)
raise MachineTranslationError('{0}: {1}'.format(
exc.__class__.__name__,
str(exc)
))
def add_unit( # noqa: C901
self,
request,
context: str,
source: Union[str, List[str]],
target: Optional[Union[str, List[str]]] = None,
extra_flags: str = "",
explanation: str = "",
auto_context: bool = False,
is_batch_update: bool = False,
):
user = request.user if request else None
component = self.component
if self.is_source:
translations = [self]
translations.extend(component.translation_set.exclude(id=self.id))
else:
translations = [component.source_translation, self]
has_template = component.has_template()
source_unit = None
result = None
# Automatic context
suffix = 0
base = context
while self.unit_set.filter(context=context, source=source).exists():
suffix += 1
context = f"{base}{suffix}"
for translation in translations:
is_source = translation.is_source
kwargs = {}
if has_template:
kwargs["pending"] = is_source
else:
kwargs["pending"] = not is_source
if kwargs["pending"]:
kwargs["details"] = {"add_unit": True}
if is_source:
current_target = source
kwargs["extra_flags"] = extra_flags
kwargs["explanation"] = explanation
else:
current_target = target
if current_target is None:
current_target = ""
if isinstance(current_target, list):
current_target = join_plural(current_target)
if isinstance(source, list):
source = join_plural(source)
if has_template:
id_hash = calculate_hash(context)
else:
id_hash = calculate_hash(source, context)
# When adding to a target the source string can already exist
unit = None
if not self.is_source and is_source:
try:
unit = translation.unit_set.get(id_hash=id_hash)
flags = Flags(unit.extra_flags)
flags.merge(extra_flags)
new_flags = flags.format()
if unit.extra_flags != new_flags or unit.explanation != explanation:
unit.extra_flags = new_flags
unit.explanation = explanation
unit.save(
update_fields=["extra_flags", "explanation"],
same_content=True,
)
except Unit.DoesNotExist:
pass
if unit is None:
unit = Unit(
translation=translation,
context=context,
source=source,
target=current_target,
state=STATE_TRANSLATED if bool(current_target) else STATE_EMPTY,
source_unit=source_unit,
id_hash=id_hash,
position=0,
**kwargs,
)
unit.is_batch_update = is_batch_update
unit.save(force_insert=True)
Change.objects.create(
unit=unit,
action=Change.ACTION_NEW_UNIT,
target=current_target,
user=user,
author=user,
)
# The source language is always first in the translations array
if source_unit is None:
source_unit = unit
if translation == self:
result = unit
if not is_batch_update:
component.update_variants()
component.sync_terminology()
return result
