def test_get_translations(gt_mock, locale_b, resource_a, google_translate_locale):
entities = [
EntityFactory(resource=resource_a, string=x, order=i)
for i, x in enumerate(["abaa", "abac", "aaab", "abab"])
]
entities[1].string_plural = entities[1].string
entities[3].string_plural = entities[3].string
entities[1].save()
entities[3].save()
google_translate_locale.cldr_plurals = "1, 2"
google_translate_locale.save()
for entity in entities[0:2]:
TranslationMemoryFactory.create(
entity=entity, source=entity.string, target=entity.string, locale=locale_b,
)
TranslationMemoryFactory.create(
entity=entity,
source=entity.string,
target=entity.string,
locale=google_translate_locale,
)
# Mock the return value of get_google_translate_data
gt_mock.return_value = {
"status": True,
"translation": "gt_translation",
}
tm_user = User.objects.get(email="*****@*****.**")
gt_user = User.objects.get(email="*****@*****.**")
# 100% match exists in translation memory.
response_a = get_translations(entities[0], locale_b)
response_b = get_translations(entities[0], google_translate_locale)
assert response_a == [(entities[0].string, None, tm_user)]
assert response_b == [(entities[0].string, None, tm_user)]
# 100% match does not exists and locale.google_translate_code is None.
response = get_translations(entities[2], locale_b)
assert response == []
# 100% match does not exists and locale.google_translate_code is not None.
response = get_translations(entities[2], google_translate_locale)
assert response == [("gt_translation", None, gt_user)]
# Entity.string_plural is not None.
response_a = get_translations(entities[1], google_translate_locale)
response_b = get_translations(entities[3], google_translate_locale)
assert response_a == [
(entities[1].string, 0, tm_user),
(entities[1].string, 1, tm_user),
]
assert response_b == [
("gt_translation", 0, gt_user),
("gt_translation", 1, gt_user),
]
def pretranslate(self, project_pk, locales=None, entities=None):
"""
Identifies strings without any translations and any suggestions.
Engages TheAlgorithm (bug 1552796) to gather pretranslations.
Stores pretranslations as suggestions (approved=False) to DB.
:arg project_pk: the pk of the project to be pretranslated
:arg Queryset locales: the locales for the project to be pretranslated
:arg Queryset entites: the entities for the project to be pretranslated
:returns: None
"""
project = Project.objects.get(pk=project_pk)
log.info("Fetching pretranslations for project {} started".format(
project.name))
if locales:
locales = project.locales.filter(pk__in=locales)
else:
locales = project.locales
locales = (locales.filter(project_locale__readonly=False).distinct().
prefetch_project_locale(project))
if not entities:
entities = Entity.objects.filter(
resource__project=project,
obsolete=False,
).prefetch_related("resource")
# get available TranslatedResource pairs
tr_pairs = (TranslatedResource.objects.filter(
resource__project=project,
locale__in=locales,
).annotate(locale_resource=Concat(
"locale_id", V("-"), "resource_id",
output_field=CharField())).values_list("locale_resource",
flat=True).distinct())
# Fetch all distinct locale-entity pairs for which translation exists
translated_entities = (Translation.objects.filter(
locale__in=locales,
entity__in=entities,
).annotate(locale_entity=Concat(
"locale_id", V("-"), "entity_id",
output_field=CharField())).values_list("locale_entity",
flat=True).distinct())
translated_entities = list(translated_entities)
translations = []
# To keep track of changed TranslatedResources and their latest_translation
tr_dict = {}
tr_filter = []
index = -1
for locale in locales:
log.info("Fetching pretranslations for locale {} started".format(
locale.code))
for entity in entities:
locale_entity = "{}-{}".format(locale.id, entity.id)
locale_resource = "{}-{}".format(locale.id, entity.resource.id)
if locale_entity in translated_entities or locale_resource not in tr_pairs:
continue
strings = get_translations(entity, locale)
if not strings:
continue
for string, plural_form, user in strings:
t = Translation(
entity=entity,
locale=locale,
string=string,
user=user,
approved=False,
fuzzy=True,
active=True,
plural_form=plural_form,
)
index += 1
translations.append(t)
if locale_resource not in tr_dict:
tr_dict[locale_resource] = index
# Add query for fetching respective TranslatedResource.
tr_filter.append(
Q(locale__id=locale.id)
& Q(resource__id=entity.resource.id))
# Update the latest translation index
tr_dict[locale_resource] = index
log.info("Fetching pretranslations for locale {} done".format(
locale.code))
if len(translations) == 0:
return
translations = Translation.objects.bulk_create(translations)
# Run checks on all translations
translation_pks = {translation.pk for translation in translations}
bulk_run_checks(
Translation.objects.for_checks().filter(pk__in=translation_pks))
# Mark translations as changed
changed_entities = {}
existing = ChangedEntityLocale.objects.values_list("entity",
"locale").distinct()
for t in translations:
key = (t.entity.pk, t.locale.pk)
# Remove duplicate changes to prevent unique constraint violation
if key not in existing:
changed_entities[key] = ChangedEntityLocale(entity=t.entity,
locale=t.locale)
ChangedEntityLocale.objects.bulk_create(changed_entities.values())
# Update latest activity and stats for changed instances.
update_changed_instances(tr_filter, tr_dict, translations)
log.info("Fetching pretranslations for project {} done".format(
project.name))
def pretranslate(project, locales=None, entities=None):
"""
Identifies strings without any translations and any suggestions.
Engages TheAlgorithm (bug 1552796) to gather pretranslations.
Stores pretranslations as suggestions (approved=False) to DB.
:arg Project project: the project to be pretranslated
:arg Queryset locales: the locales for the project to be pretranslated
:arg Queryset entites: the entities for the project to be pretranslated
:returns: None
"""
log.info("Fetching pretranslations for project {} started".format(
project.name))
if locales:
locales = project.locales.filter(pk__in=locales)
else:
locales = project.locales
locales = (locales.filter(project_locale__readonly=False).distinct().
prefetch_project_locale(project))
if not entities:
entities = Entity.objects.filter(
resource__project=project,
obsolete=False,
).prefetch_related("resource")
# get available TranslatedResource pairs
tr_pairs = (TranslatedResource.objects.filter(
resource__project=project,
locale__in=locales,
).annotate(locale_resource=Concat(
"locale_id", V("-"), "resource_id",
output_field=CharField())).values_list("locale_resource",
flat=True).distinct())
# Fetch all distinct locale-entity pairs for which translation exists
translated_entities = (Translation.objects.filter(
locale__in=locales,
entity__in=entities,
).annotate(locale_entity=Concat(
"locale_id", V("-"), "entity_id",
output_field=CharField())).values_list("locale_entity",
flat=True).distinct())
translated_entities = list(translated_entities)
translations = []
# To keep track of changed Locales and TranslatedResources
# Also, their latest_translation and stats count
locale_dict = {}
tr_dict = {}
tr_filter = []
index = -1
for locale in locales:
log.info("Fetching pretranslations for locale {} started".format(
locale.code))
for entity in entities:
locale_entity = "{}-{}".format(locale.id, entity.id)
locale_resource = "{}-{}".format(locale.id, entity.resource.id)
if locale_entity in translated_entities or locale_resource not in tr_pairs:
continue
strings = get_translations(entity, locale)
if not strings:
continue
for string, plural_form, user in strings:
t = Translation(
entity=entity,
locale=locale,
string=string,
user=user,
approved=False,
active=True,
plural_form=plural_form,
)
translations.append(t)
index += 1
if locale_resource not in tr_dict:
tr_dict[locale_resource] = [index, 0]
# Add query for fetching respective TranslatedResource.
tr_filter.append(
Q(locale__id=locale.id)
& Q(resource__id=entity.resource.id))
if locale.code not in locale_dict:
locale_dict[locale.code] = [locale, index, 0]
# Update the latest translation index
tr_dict[locale_resource][0] = index
locale_dict[locale.code][1] = index
# Increment number of translations (used to adjust stats)
tr_dict[locale_resource][1] += 1
locale_dict[locale.code][2] += 1
log.info("Fetching pretranslations for locale {} done".format(
locale.code))
if len(translations) == 0:
return
translations = Translation.objects.bulk_create(translations)
# Update latest activity and unreviewed count for the project.
project.latest_translation = translations[-1]
project.unreviewed_strings += len(translations)
project.save(update_fields=["latest_translation", "unreviewed_strings"])
# Update latest activity and unreviewed count for changed instances.
update_changed_instances(tr_filter, tr_dict, locale_dict, translations)
log.info("Fetching pretranslations for project {} done".format(
project.name))