def manually_pretranslate_project(request, slug):
if not request.user.has_perm("base.can_manage_project"):
return HttpResponseForbidden(
"Forbidden: You don't have permission for pretranslating projects"
)
project = Project.objects.get(slug=slug)
pretranslate(project)
return HttpResponse("ok")
def test_pretranslate(gt_mock, project_a, locale_a, resource_a, locale_b):
project_a.pretranslation_enabled = True
project_a.save()
resources = [
ResourceFactory(project=project_a, path=x, format="po")
for x in ["resource_x.po", "resource_y.po"]
]
for i, x in enumerate(["abaa", "abac"]):
EntityFactory.create(resource=resources[0], string=x, order=i)
for i, x in enumerate(["aaab", "abab"]):
EntityFactory.create(resource=resources[1], string=x, order=i)
TranslatedResourceFactory.create(resource=resources[0], locale=locale_a)
TranslatedResourceFactory.create(resource=resources[0], locale=locale_b)
TranslatedResourceFactory.create(resource=resources[1], locale=locale_a)
ProjectLocaleFactory.create(
project=project_a,
locale=locale_a,
pretranslation_enabled=True,
)
ProjectLocaleFactory.create(
project=project_a,
locale=locale_b,
pretranslation_enabled=True,
)
tm_user = User.objects.get(email="*****@*****.**")
gt_mock.return_value = [("pretranslation", None, tm_user)]
pretranslate(project_a.pk)
project_a.refresh_from_db()
translations = Translation.objects.filter(user=tm_user)
# Total pretranslations = 2(tr_ax) + 2(tr_bx) + 2(tr_ay)
assert len(translations) == 6
# pretranslated count == total pretranslations.
assert project_a.pretranslated_strings == 6
# latest_translation belongs to pretranslations.
assert project_a.latest_translation in translations
def sync_translations(
self,
project_pk,
project_sync_log_pk,
now,
added_paths=None,
removed_paths=None,
changed_paths=None,
new_entities=None,
locale=None,
no_pull=False,
no_commit=False,
full_scan=False,
):
db_project = get_or_fail(
Project,
pk=project_pk,
message="Could not sync project with pk={0}, not found.".format(project_pk),
)
repos = db_project.translation_repositories()
repo_pk = repos[0].pk
repo = get_or_fail(
Repository,
pk=repo_pk,
message="Could not sync repo with pk={0}, not found.".format(repo_pk),
)
project_sync_log = get_or_fail(
ProjectSyncLog,
pk=project_sync_log_pk,
message=(
"Could not sync project {0}, log with pk={1} not found.".format(
db_project.slug, project_sync_log_pk
)
),
)
log.info("Syncing translations for project: {}".format(db_project.slug))
repo_sync_log = RepositorySyncLog.objects.create(
project_sync_log=project_sync_log, repository=repo, start_time=timezone.now()
)
if locale:
locales = db_project.locales.filter(pk=locale.pk)
else:
locales = db_project.locales.all()
if not locales:
log.info(
"Skipping syncing translations for project {0}, no locales to sync "
"found within.".format(db_project.slug)
)
repo_sync_log.end()
return
# If project repositories have API access, we can retrieve latest commit hashes and detect
# changed locales before the expensive VCS pull/clone operations. When performing full scan,
# we still need to sync all locales.
if not full_scan:
locales = get_changed_locales(db_project, locales, now)
readonly_locales = db_project.locales.filter(project_locale__readonly=True)
added_and_changed_resources = db_project.resources.filter(
path__in=list(added_paths or []) + list(changed_paths or [])
).distinct()
# We should also sync files for which source file change - but only for read-only locales.
# See bug 1372151 for more details.
if added_and_changed_resources:
changed_locales_pks = [l.pk for l in locales]
readonly_locales_pks = [l.pk for l in readonly_locales]
locales = db_project.locales.filter(
pk__in=changed_locales_pks + readonly_locales_pks
)
# Pull VCS changes in case we're on a different worker than the one
# sync started on.
if not no_pull:
log.info("Pulling changes for project {0} started.".format(db_project.slug))
repos_changed, repo_locales = pull_changes(db_project, locales)
repos = repos.filter(pk__in=repo_locales.keys())
log.info("Pulling changes for project {0} complete.".format(db_project.slug))
# If none of the repos has changed since the last sync and there are
# no Pontoon-side changes for this project, quit early.
if (
not full_scan
and not db_project.needs_sync
and not repos_changed
and not (added_paths or removed_paths or changed_paths)
):
log.info("Skipping project {0}, no changes detected.".format(db_project.slug))
repo_sync_log.end()
return
vcs_project = VCSProject(
db_project,
now,
locales=locales,
repo_locales=repo_locales,
added_paths=added_paths,
changed_paths=changed_paths,
full_scan=full_scan,
)
synced_locales = set()
failed_locales = set()
# Store newly added locales and locales with newly added resources
new_locales = []
for locale in locales:
try:
with transaction.atomic():
# Sets VCSProject.synced_locales, needed to skip early
if not vcs_project.synced_locales:
vcs_project.resources
# Skip all locales if none of the them has anything to sync
if len(vcs_project.synced_locales) == 0:
break
# Skip locales that have nothing to sync
if (
vcs_project.synced_locales
and locale not in vcs_project.synced_locales
):
continue
changeset = ChangeSet(db_project, vcs_project, now, locale)
update_translations(db_project, vcs_project, locale, changeset)
changeset.execute()
created = update_translated_resources(db_project, vcs_project, locale)
if created:
new_locales.append(locale.pk)
update_locale_project_locale_stats(locale, db_project)
# Clear out the "has_changed" markers now that we've finished
# syncing.
(
ChangedEntityLocale.objects.filter(
entity__resource__project=db_project,
locale=locale,
when__lte=now,
).delete()
)
# Perform the commit last so that, if it succeeds, there is
# nothing after it to fail.
if (
not no_commit
and locale in changeset.locales_to_commit
and locale not in readonly_locales
):
commit_changes(db_project, vcs_project, changeset, locale)
log.info(
"Synced locale {locale} for project {project}.".format(
locale=locale.code, project=db_project.slug,
)
)
synced_locales.add(locale.code)
except CommitToRepositoryException as err:
# Transaction aborted, log and move on to the next locale.
log.warning(
"Failed to sync locale {locale} for project {project} due to "
"commit error: {error}".format(
locale=locale.code, project=db_project.slug, error=err,
)
)
failed_locales.add(locale.code)
# If sources have changed, update stats for all locales.
if added_paths or removed_paths or changed_paths:
for locale in db_project.locales.all():
# Already synced.
if locale.code in synced_locales:
continue
# We have files: update all translated resources.
if locale in locales:
created = update_translated_resources(db_project, vcs_project, locale)
if created:
new_locales.append[locale.pk]
# We don't have files: we can still update asymmetric translated resources.
else:
update_translated_resources_no_files(
db_project, locale, added_and_changed_resources,
)
update_locale_project_locale_stats(locale, db_project)
synced_locales.add(locale.code)
log.info(
"Synced source changes for locale {locale} for project {project}.".format(
locale=locale.code, project=db_project.slug,
)
)
db_project.aggregate_stats()
if synced_locales:
log.info(
"Synced translations for project {0} in locales {1}.".format(
db_project.slug, ",".join(synced_locales)
)
)
elif failed_locales:
log.info(
"Failed to sync translations for project {0} due to commit error.".format(
db_project.slug
)
)
else:
log.info(
"Skipping syncing translations for project {0}, none of the locales "
"has anything to sync.".format(db_project.slug)
)
for r in repos:
r.set_last_synced_revisions(
locales=repo_locales[r.pk].exclude(code__in=failed_locales)
)
repo_sync_log.end()
if db_project.pretranslation_enabled:
# Pretranslate all entities for newly added locales
# and locales with newly added resources
if len(new_locales):
pretranslate(db_project, locales=new_locales)
locales = db_project.locales.exclude(pk__in=new_locales).values_list(
"pk", flat=True
)
# Pretranslate newly added entities for all locales
if new_entities and locales:
new_entities = list(set(new_entities))
pretranslate(db_project, locales=locales, entities=new_entities)