def bulk_check_translations(self):
"""
Run checks on all changed translations from supported resources
:return: primary keys of translations without warnings and errors.
"""
changed_pks = {t.pk for t in self.changed_translations}
bulk_run_checks(
Translation.objects
.for_checks()
.filter(pk__in=changed_pks)
)
valid_translations = set(
Translation.objects
.filter(
pk__in=changed_pks,
errors__isnull=True,
)
.values_list('pk', flat=True)
.order_by('pk')
)
return valid_translations
def bulk_check_translations(self):
"""
Run checks on all changed translations from supported resources
:return: primary keys of translations without warnings and errors.
"""
changed_pks = {t.pk for t in self.changed_translations}
bulk_run_checks(
Translation.objects
.for_checks()
.filter(pk__in=changed_pks)
)
valid_translations = set(
Translation.objects
.filter(
pk__in=changed_pks,
errors__isnull=True,
)
.values_list('pk', flat=True)
.order_by('pk')
)
return valid_translations
def test_bulk_run_checks(
translation_compare_locales_warning,
translation_compare_locales_error,
translation_pontoon_error,
):
warnings, errors = bulk_run_checks([
translation_compare_locales_warning,
translation_pontoon_error,
translation_compare_locales_error,
])
(cl_warning, ) = warnings
p_error, cl_error = errors
assert cl_warning.pk is not None
assert cl_warning.library == "cl"
assert cl_warning.message == "unknown escape sequence, \\q"
assert cl_warning.translation == translation_compare_locales_warning
assert cl_error.pk is not None
assert cl_error.library == "cl"
assert cl_error.message == "Found single %"
assert cl_error.translation == translation_compare_locales_error
assert p_error.pk is not None
assert p_error.library == "p"
assert p_error.message == "Empty translations are not allowed"
assert p_error.translation == translation_pontoon_error
def test_bulk_run_checks(
translation_compare_locales_warning,
translation_compare_locales_error,
translation_pontoon_error,
):
warnings, errors = bulk_run_checks([
translation_compare_locales_warning,
translation_pontoon_error,
translation_compare_locales_error,
])
cl_warning, = warnings
p_error, cl_error = errors
assert cl_warning.pk is not None
assert cl_warning.library == 'cl'
assert cl_warning.message == 'unknown escape sequence, \q'
assert cl_warning.translation == translation_compare_locales_warning
assert cl_error.pk is not None
assert cl_error.library == 'cl'
assert cl_error.message == 'Found single %'
assert cl_error.translation == translation_compare_locales_error
assert p_error.pk is not None
assert p_error.library == 'p'
assert p_error.message == 'Empty translations are not allowed'
assert p_error.translation == translation_pontoon_error
def translation_dtd_unapproved():
translation = TranslationFactory.create(
string='Test Translation',
approved=False,
entity__key='test',
entity__resource__format='dtd',
entity__resource__path='test.dtd',
)
bulk_run_checks([translation])
ProjectLocaleFactory.create(
project=translation.entity.resource.project,
locale=translation.locale,
)
yield translation
def test_bulk_run_checks(
translation_compare_locales_warning,
translation_compare_locales_error,
translation_pontoon_error,
):
warnings, errors = bulk_run_checks([
translation_compare_locales_warning,
translation_pontoon_error,
translation_compare_locales_error,
])
cl_warning, = warnings
p_error, cl_error = errors
assert cl_warning.pk is not None
assert cl_warning.library == 'cl'
assert cl_warning.message == 'unknown escape sequence, \q'
assert cl_warning.translation == translation_compare_locales_warning
assert cl_error.pk is not None
assert cl_error.library == 'cl'
assert cl_error.message == 'Found single %'
assert cl_error.translation == translation_compare_locales_error
assert p_error.pk is not None
assert p_error.library == 'p'
assert p_error.message == 'Empty translations are not allowed'
assert p_error.translation == translation_pontoon_error
def translation_dtd_unapproved():
translation = TranslationFactory.create(
string="Test Translation",
active=True,
approved=False,
entity__key="test",
entity__resource__format="dtd",
entity__resource__path="test.dtd",
)
bulk_run_checks([translation])
ProjectLocaleFactory.create(
project=translation.entity.resource.project,
locale=translation.locale,
)
yield translation
def test_bulk_run_checks_no_translations():
"""
Don't generate any warnings and errors without translations
"""
assert bulk_run_checks([]) is None
assert Warning.objects.count() == 0
assert Error.objects.count() == 0
def translation_dtd_invalid_unapproved():
# Provide invalid characters in translation to cause checks to fail
translation = TranslationFactory.create(
string='!@#$""\'',
approved=False,
entity__key='test',
entity__resource__format='dtd',
entity__resource__path='test.dtd',
)
bulk_run_checks([translation])
ProjectLocaleFactory.create(
project=translation.entity.resource.project,
locale=translation.locale,
)
yield translation
def test_bulk_run_checks_no_translations():
"""
Don't generate any warnings and errors without translations
"""
assert bulk_run_checks([]) is None
assert Warning.objects.count() == 0
assert Error.objects.count() == 0
def check_translations(self, translations_pks):
"""
Run checks on translations
:arg list[int] translations_pks: list of primary keys for translations that should be processed
"""
with transaction.atomic():
translations = (Translation.objects.for_checks().filter(
pk__in=translations_pks))
warnings, errors = bulk_run_checks(translations)
log.info(
"Task: {}, Processed items: {}, Warnings: {}, Errors: {}".format(
self.request.id,
len(translations),
len(warnings),
len(errors),
))
def check_translations(self, translations_pks):
"""
Run checks on translations
:arg list[int] translations_pks: list of primary keys for translations that should be processed
"""
start_time = time.time()
with transaction.atomic():
translations = get_translations(pk__in=translations_pks)
warnings, errors = bulk_run_checks(translations)
log.info("Task[{}]: Processed items: {}, Warnings({}) Errors({}) in {}".format(
self.request.id,
len(translations),
len(warnings),
len(errors),
time.time() - start_time
))
def check_translations(self, translations_pks):
"""
Run checks on translations
:arg list[int] translations_pks: list of primary keys for translations that should be processed
"""
with transaction.atomic():
translations = (
Translation.objects
.for_checks()
.filter(pk__in=translations_pks)
)
warnings, errors = bulk_run_checks(translations)
log.info("Task: {}, Processed items: {}, Warnings: {}, Errors: {}".format(
self.request.id,
len(translations),
len(warnings),
len(errors),
))
def pretranslate(self, project_pk, locales=None, entities=None):
"""
Identifies strings without any translations and any suggestions.
Engages TheAlgorithm (bug 1552796) to gather pretranslations.
Stores pretranslations as suggestions (approved=False) to DB.
:arg project_pk: the pk of the project to be pretranslated
:arg Queryset locales: the locales for the project to be pretranslated
:arg Queryset entites: the entities for the project to be pretranslated
:returns: None
"""
project = Project.objects.get(pk=project_pk)
log.info("Fetching pretranslations for project {} started".format(
project.name))
if locales:
locales = project.locales.filter(pk__in=locales)
else:
locales = project.locales
locales = (locales.filter(project_locale__readonly=False).distinct().
prefetch_project_locale(project))
if not entities:
entities = Entity.objects.filter(
resource__project=project,
obsolete=False,
).prefetch_related("resource")
# get available TranslatedResource pairs
tr_pairs = (TranslatedResource.objects.filter(
resource__project=project,
locale__in=locales,
).annotate(locale_resource=Concat(
"locale_id", V("-"), "resource_id",
output_field=CharField())).values_list("locale_resource",
flat=True).distinct())
# Fetch all distinct locale-entity pairs for which translation exists
translated_entities = (Translation.objects.filter(
locale__in=locales,
entity__in=entities,
).annotate(locale_entity=Concat(
"locale_id", V("-"), "entity_id",
output_field=CharField())).values_list("locale_entity",
flat=True).distinct())
translated_entities = list(translated_entities)
translations = []
# To keep track of changed TranslatedResources and their latest_translation
tr_dict = {}
tr_filter = []
index = -1
for locale in locales:
log.info("Fetching pretranslations for locale {} started".format(
locale.code))
for entity in entities:
locale_entity = "{}-{}".format(locale.id, entity.id)
locale_resource = "{}-{}".format(locale.id, entity.resource.id)
if locale_entity in translated_entities or locale_resource not in tr_pairs:
continue
strings = get_translations(entity, locale)
if not strings:
continue
for string, plural_form, user in strings:
t = Translation(
entity=entity,
locale=locale,
string=string,
user=user,
approved=False,
fuzzy=True,
active=True,
plural_form=plural_form,
)
index += 1
translations.append(t)
if locale_resource not in tr_dict:
tr_dict[locale_resource] = index
# Add query for fetching respective TranslatedResource.
tr_filter.append(
Q(locale__id=locale.id)
& Q(resource__id=entity.resource.id))
# Update the latest translation index
tr_dict[locale_resource] = index
log.info("Fetching pretranslations for locale {} done".format(
locale.code))
if len(translations) == 0:
return
translations = Translation.objects.bulk_create(translations)
# Run checks on all translations
translation_pks = {translation.pk for translation in translations}
bulk_run_checks(
Translation.objects.for_checks().filter(pk__in=translation_pks))
# Mark translations as changed
changed_entities = {}
existing = ChangedEntityLocale.objects.values_list("entity",
"locale").distinct()
for t in translations:
key = (t.entity.pk, t.locale.pk)
# Remove duplicate changes to prevent unique constraint violation
if key not in existing:
changed_entities[key] = ChangedEntityLocale(entity=t.entity,
locale=t.locale)
ChangedEntityLocale.objects.bulk_create(changed_entities.values())
# Update latest activity and stats for changed instances.
update_changed_instances(tr_filter, tr_dict, translations)
log.info("Fetching pretranslations for project {} done".format(
project.name))
