def sync_resources(db_project, now, force, no_pull):
# Pull source repository
if no_pull:
source_repo_changed = True # Assume changed
else:
source_repo_changed = pull_changes(db_project, source_only=True)
# If the only repo hasn't changed since the last sync and there are
# no Pontoon-side changes for this project, quit early.
if not force and not db_project.needs_sync and not source_repo_changed and db_project.has_single_repo:
log.info('Skipping project {0}, no changes detected.'.format(db_project.slug))
return False
if force or source_repo_changed:
try:
project_changes, obsolete_vcs_resources, new_paths = update_originals(db_project, now, full_scan=force)
except MissingSourceDirectoryError as e:
log.error(e)
return False
if not db_project.has_single_repo:
db_project.source_repository.set_last_synced_revisions()
log.info('Synced resources for project {0}.'.format(db_project.slug))
else:
project_changes, obsolete_vcs_resources, new_paths = None, None, None
log.info('Skipping syncing resources for project {0}, no changes detected.'.format(db_project.slug))
return {
'project_changes': project_changes,
'obsolete_vcs_resources': obsolete_vcs_resources,
'new_paths': new_paths,
}
def sync_resources(db_project, now, force, no_pull):
# Pull source repository
if no_pull:
source_repo_changed = True # Assume changed
else:
source_repo_changed, _ = pull_changes(db_project, source_only=True)
# If the only repo hasn't changed since the last sync and there are
# no Pontoon-side changes for this project, quit early.
if not force and not db_project.needs_sync and not source_repo_changed and db_project.has_single_repo:
log.info('Skipping project {0}, no changes detected.'.format(db_project.slug))
return False
if force or source_repo_changed:
try:
project_changes, obsolete_vcs_resources, new_paths = update_originals(db_project, now, full_scan=force)
except MissingSourceDirectoryError as e:
log.error(e)
return False
if not db_project.has_single_repo:
db_project.source_repository.set_last_synced_revisions()
log.info('Synced resources for project {0}.'.format(db_project.slug))
else:
project_changes, obsolete_vcs_resources, new_paths = None, None, None
log.info('Skipping syncing resources for project {0}, no changes detected.'.format(db_project.slug))
return {
'project_changes': project_changes,
'obsolete_vcs_resources': obsolete_vcs_resources,
'new_paths': new_paths,
}
def sync_project(self,
project_pk,
sync_log_pk,
no_pull=False,
no_commit=False,
force=False):
"""Fetch the project with the given PK and perform sync on it."""
db_project = get_or_fail(
Project,
pk=project_pk,
message='Could not sync project with pk={0}, not found.'.format(
project_pk))
sync_log = get_or_fail(
SyncLog,
pk=sync_log_pk,
message=(
'Could not sync project {0}, log with pk={1} not found.'.format(
db_project.slug, sync_log_pk)))
log.info('Syncing project {0}.'.format(db_project.slug))
# Mark "now" at the start of sync to avoid messing with
# translations submitted during sync.
now = timezone.now()
project_sync_log = ProjectSyncLog.objects.create(sync_log=sync_log,
project=db_project,
start_time=now)
if not no_pull:
repos_changed = pull_changes(db_project)
else:
repos_changed = True # Assume changed.
# If the repos haven't changed since the last sync and there are
# no Pontoon-side changes for this project, quit early.
if not force and not repos_changed and not db_project.needs_sync:
log.info('Skipping project {0}, no changes detected.'.format(
db_project.slug))
project_sync_log.skipped = True
project_sync_log.skipped_end_time = timezone.now()
project_sync_log.save(update_fields=('skipped', 'skipped_end_time'))
return
obsolete_vcs = perform_sync_project(db_project, now)
for repo in db_project.repositories.all():
sync_project_repo.delay(project_pk,
repo.pk,
project_sync_log.pk,
now,
obsolete_vcs,
no_pull=no_pull,
no_commit=no_commit)
log.info('Synced resources for project {0}.'.format(db_project.slug))
def test_unsure_changes(self):
"""
If any of the repos returns None as a revision number, consider
the VCS as changed even if the revisions match the last sync.
"""
self.mock_repo_pull.return_value = {'single_locale': None}
self.repository.last_synced_revisions = {'single_locale': None}
self.repository.save()
assert_true(pull_changes(self.db_project))
def test_basic(self):
"""
Pull_changes should call repo.pull for each repo for the
project, save the return value to repo.last_synced_revisions,
and return whether any changes happened in VCS.
"""
self.mock_repo_pull.return_value = {'single_locale': 'asdf'}
assert_true(pull_changes(self.db_project))
self.repository.refresh_from_db()
assert_equal(self.repository.last_synced_revisions, {'single_locale': 'asdf'})
def test_unchanged(self):
"""
If the revisions returned by repo.pull match those from the last
sync, consider the VCS unchanged and return False.
"""
self.mock_repo_pull.return_value = {'single_locale': 'asdf'}
self.repository.last_synced_revisions = {'single_locale': 'asdf'}
self.repository.save()
assert_false(pull_changes(self.db_project))
def test_unchanged(self):
"""
If the revisions returned by repo.pull match those from the last
sync, consider the VCS unchanged and return False.
"""
self.mock_repo_pull.return_value = {'single_locale': 'asdf'}
self.repository.last_synced_revisions = {'single_locale': 'asdf'}
self.repository.save()
has_changed, _ = pull_changes(self.db_project, locales=self.db_project.locales.all())
assert_false(has_changed)
def test_unsure_changes(self):
"""
If any of the repos returns None as a revision number, consider
the VCS as changed even if the revisions match the last sync.
"""
self.mock_repo_pull.return_value = {'single_locale': None}
self.repository.last_synced_revisions = {'single_locale': None}
self.repository.save()
assert_true(pull_changes(self.db_project))
def test_basic(self):
"""
Pull_changes should call repo.pull for each repo for the
project, save the return value to repo.last_synced_revisions,
and return whether any changes happened in VCS.
"""
self.mock_repo_pull.return_value = {'single_locale': 'asdf'}
assert_true(pull_changes(self.db_project))
self.repository.refresh_from_db()
assert_equal(self.repository.last_synced_revisions, {'single_locale': 'asdf'})
def test_basic(self):
"""
Pull_changes should call repo.pull for each repo for the
project and return whether any changes happened in VCS.
"""
mock_db_project = MagicMock()
mock_db_project.repositories.all.return_value = [self.repository]
self.mock_repo_pull.return_value = {'single_locale': 'asdf'}
has_changed = pull_changes(self.db_project)
assert_true(has_changed)
def test_basic(self):
"""
Pull_changes should call repo.pull for each repo for the
project and return whether any changes happened in VCS.
"""
mock_db_project = MagicMock()
mock_db_project.repositories.all.return_value = [self.repository]
self.mock_repo_pull.return_value = {'single_locale': 'asdf'}
has_changed, _ = pull_changes(self.db_project)
assert_true(has_changed)
def sync_project(self, project_pk, sync_log_pk, no_pull=False, no_commit=False, force=False):
"""Fetch the project with the given PK and perform sync on it."""
db_project = get_or_fail(Project, pk=project_pk,
message='Could not sync project with pk={0}, not found.'.format(project_pk))
sync_log = get_or_fail(SyncLog, pk=sync_log_pk,
message=('Could not sync project {0}, log with pk={1} not found.'
.format(db_project.slug, sync_log_pk)))
log.info('Syncing project {0}.'.format(db_project.slug))
# Mark "now" at the start of sync to avoid messing with
# translations submitted during sync.
now = timezone.now()
project_sync_log = ProjectSyncLog.objects.create(
sync_log=sync_log,
project=db_project,
start_time=now
)
if not no_pull:
repos_changed = pull_changes(db_project)
else:
repos_changed = True # Assume changed.
# If the repos haven't changed since the last sync and there are
# no Pontoon-side changes for this project, quit early.
if not force and not repos_changed and not db_project.needs_sync:
log.info('Skipping project {0}, no changes detected.'.format(db_project.slug))
project_sync_log.skipped = True
project_sync_log.skipped_end_time = timezone.now()
project_sync_log.save(update_fields=('skipped', 'skipped_end_time'))
return
obsolete_vcs = perform_sync_project(db_project, now)
for repo in db_project.repositories.all():
sync_project_repo.delay(
project_pk,
repo.pk,
project_sync_log.pk,
now,
obsolete_vcs,
no_pull=no_pull,
no_commit=no_commit
)
log.info('Synced resources for project {0}.'.format(db_project.slug))
def test_basic(self):
"""
Pull_changes should call repo.pull for each repo for the
project, save the return value to repo.last_synced_revisions,
and return whether any changes happened in VCS.
"""
mock_db_project = MagicMock()
mock_db_project.repositories.all.return_value = [self.repository]
self.mock_repo_pull.return_value = {'single_locale': 'asdf'}
has_changed, revisions = pull_changes(self.db_project)
assert_true(has_changed)
assert_equal(revisions, {self.repository.pk: {'single_locale': 'asdf'}})
self.repository.last_synced_revisions = revisions[self.repository.pk]
self.repository.save()
self.repository.refresh_from_db()
assert_equal(self.repository.last_synced_revisions, {'single_locale': 'asdf'})
def sync_sources(db_project, now, force, no_pull):
# Pull source repository
if no_pull:
source_repo_changed = True # Assume changed
else:
source_repo_changed, _ = pull_changes(db_project)
# If the only repo hasn't changed since the last sync and there are
# no Pontoon-side changes for this project, quit early.
if (
not force
and not db_project.needs_sync
and not source_repo_changed
and db_project.has_single_repo
):
log.info("Skipping project {0}, no changes detected.".format(db_project.slug))
return False
if force or source_repo_changed:
try:
added_paths, removed_paths, changed_paths, new_entities = update_originals(
db_project, now, full_scan=force
)
except MissingSourceDirectoryError as e:
log.error(e)
return False
if not db_project.has_single_repo:
db_project.source_repository.set_last_synced_revisions()
log.info("Synced sources for project {0}.".format(db_project.slug))
else:
added_paths, removed_paths, changed_paths, new_entities = None, None, None, None
log.info(
"Skipping syncing sources for project {0}, no changes detected.".format(
db_project.slug
)
)
return {
"added_paths": added_paths,
"removed_paths": removed_paths,
"changed_paths": changed_paths,
"new_entities": new_entities,
}
def test_basic(self):
"""
Pull_changes should call repo.pull for each repo for the
project, save the return value to repo.last_synced_revisions,
and return whether any changes happened in VCS.
"""
mock_db_project = MagicMock()
mock_db_project.repositories.all.return_value = [self.repository]
self.mock_repo_pull.return_value = {'single_locale': 'asdf'}
has_changed, revisions = pull_changes(self.db_project)
assert_true(has_changed)
assert_equal(revisions,
{self.repository.pk: {
'single_locale': 'asdf'
}})
self.repository.last_synced_revisions = revisions[self.repository.pk]
self.repository.save()
self.repository.refresh_from_db()
assert_equal(self.repository.last_synced_revisions,
{'single_locale': 'asdf'})
def sync_project(self, project_pk, sync_log_pk, no_pull=False, no_commit=False, force=False):
"""Fetch the project with the given PK and perform sync on it."""
db_project = get_or_fail(Project, pk=project_pk,
message='Could not sync project with pk={0}, not found.'.format(project_pk))
sync_log = get_or_fail(SyncLog, pk=sync_log_pk,
message=('Could not sync project {0}, log with pk={1} not found.'
.format(db_project.slug, sync_log_pk)))
log.info('Syncing project {0}.'.format(db_project.slug))
# Mark "now" at the start of sync to avoid messing with
# translations submitted during sync.
now = timezone.now()
project_sync_log = ProjectSyncLog.objects.create(
sync_log=sync_log,
project=db_project,
start_time=now
)
if not no_pull:
repos_changed = pull_changes(db_project)
else:
repos_changed = True # Assume changed.
# If the repos haven't changed since the last sync and there are
# no Pontoon-side changes for this project, quit early.
if not force and not repos_changed and not db_project.needs_sync:
log.info('Skipping project {0}, no changes detected.'.format(db_project.slug))
project_sync_log.skip()
return
try:
obsolete_vcs_entities, obsolete_vcs_resources = perform_sync_project(db_project, now)
except MissingSourceDirectoryError, e:
log.error(e)
project_sync_log.skip()
return
def sync_translations(
self,
project_pk,
project_sync_log_pk,
now,
added_paths=None,
removed_paths=None,
changed_paths=None,
new_entities=None,
locale=None,
no_pull=False,
no_commit=False,
full_scan=False,
):
db_project = get_or_fail(
Project,
pk=project_pk,
message="Could not sync project with pk={0}, not found.".format(project_pk),
)
repos = db_project.translation_repositories()
repo_pk = repos[0].pk
repo = get_or_fail(
Repository,
pk=repo_pk,
message="Could not sync repo with pk={0}, not found.".format(repo_pk),
)
project_sync_log = get_or_fail(
ProjectSyncLog,
pk=project_sync_log_pk,
message=(
"Could not sync project {0}, log with pk={1} not found.".format(
db_project.slug, project_sync_log_pk
)
),
)
log.info("Syncing translations for project: {}".format(db_project.slug))
repo_sync_log = RepositorySyncLog.objects.create(
project_sync_log=project_sync_log, repository=repo, start_time=timezone.now()
)
if locale:
locales = db_project.locales.filter(pk=locale.pk)
else:
locales = db_project.locales.all()
if not locales:
log.info(
"Skipping syncing translations for project {0}, no locales to sync "
"found within.".format(db_project.slug)
)
repo_sync_log.end()
return
# If project repositories have API access, we can retrieve latest commit hashes and detect
# changed locales before the expensive VCS pull/clone operations. When performing full scan,
# we still need to sync all locales.
if not full_scan:
locales = get_changed_locales(db_project, locales, now)
readonly_locales = db_project.locales.filter(project_locale__readonly=True)
added_and_changed_resources = db_project.resources.filter(
path__in=list(added_paths or []) + list(changed_paths or [])
).distinct()
# We should also sync files for which source file change - but only for read-only locales.
# See bug 1372151 for more details.
if added_and_changed_resources:
changed_locales_pks = [l.pk for l in locales]
readonly_locales_pks = [l.pk for l in readonly_locales]
locales = db_project.locales.filter(
pk__in=changed_locales_pks + readonly_locales_pks
)
# Pull VCS changes in case we're on a different worker than the one
# sync started on.
if not no_pull:
log.info("Pulling changes for project {0} started.".format(db_project.slug))
repos_changed, repo_locales = pull_changes(db_project, locales)
repos = repos.filter(pk__in=repo_locales.keys())
log.info("Pulling changes for project {0} complete.".format(db_project.slug))
# If none of the repos has changed since the last sync and there are
# no Pontoon-side changes for this project, quit early.
if (
not full_scan
and not db_project.needs_sync
and not repos_changed
and not (added_paths or removed_paths or changed_paths)
):
log.info("Skipping project {0}, no changes detected.".format(db_project.slug))
repo_sync_log.end()
return
vcs_project = VCSProject(
db_project,
now,
locales=locales,
repo_locales=repo_locales,
added_paths=added_paths,
changed_paths=changed_paths,
full_scan=full_scan,
)
synced_locales = set()
failed_locales = set()
# Store newly added locales and locales with newly added resources
new_locales = []
for locale in locales:
try:
with transaction.atomic():
# Sets VCSProject.synced_locales, needed to skip early
if not vcs_project.synced_locales:
vcs_project.resources
# Skip all locales if none of the them has anything to sync
if len(vcs_project.synced_locales) == 0:
break
# Skip locales that have nothing to sync
if (
vcs_project.synced_locales
and locale not in vcs_project.synced_locales
):
continue
changeset = ChangeSet(db_project, vcs_project, now, locale)
update_translations(db_project, vcs_project, locale, changeset)
changeset.execute()
created = update_translated_resources(db_project, vcs_project, locale)
if created:
new_locales.append(locale.pk)
update_locale_project_locale_stats(locale, db_project)
# Clear out the "has_changed" markers now that we've finished
# syncing.
(
ChangedEntityLocale.objects.filter(
entity__resource__project=db_project,
locale=locale,
when__lte=now,
).delete()
)
# Perform the commit last so that, if it succeeds, there is
# nothing after it to fail.
if (
not no_commit
and locale in changeset.locales_to_commit
and locale not in readonly_locales
):
commit_changes(db_project, vcs_project, changeset, locale)
log.info(
"Synced locale {locale} for project {project}.".format(
locale=locale.code, project=db_project.slug,
)
)
synced_locales.add(locale.code)
except CommitToRepositoryException as err:
# Transaction aborted, log and move on to the next locale.
log.warning(
"Failed to sync locale {locale} for project {project} due to "
"commit error: {error}".format(
locale=locale.code, project=db_project.slug, error=err,
)
)
failed_locales.add(locale.code)
# If sources have changed, update stats for all locales.
if added_paths or removed_paths or changed_paths:
for locale in db_project.locales.all():
# Already synced.
if locale.code in synced_locales:
continue
# We have files: update all translated resources.
if locale in locales:
created = update_translated_resources(db_project, vcs_project, locale)
if created:
new_locales.append[locale.pk]
# We don't have files: we can still update asymmetric translated resources.
else:
update_translated_resources_no_files(
db_project, locale, added_and_changed_resources,
)
update_locale_project_locale_stats(locale, db_project)
synced_locales.add(locale.code)
log.info(
"Synced source changes for locale {locale} for project {project}.".format(
locale=locale.code, project=db_project.slug,
)
)
db_project.aggregate_stats()
if synced_locales:
log.info(
"Synced translations for project {0} in locales {1}.".format(
db_project.slug, ",".join(synced_locales)
)
)
elif failed_locales:
log.info(
"Failed to sync translations for project {0} due to commit error.".format(
db_project.slug
)
)
else:
log.info(
"Skipping syncing translations for project {0}, none of the locales "
"has anything to sync.".format(db_project.slug)
)
for r in repos:
r.set_last_synced_revisions(
locales=repo_locales[r.pk].exclude(code__in=failed_locales)
)
repo_sync_log.end()
if db_project.pretranslation_enabled:
# Pretranslate all entities for newly added locales
# and locales with newly added resources
if len(new_locales):
pretranslate(db_project, locales=new_locales)
locales = db_project.locales.exclude(pk__in=new_locales).values_list(
"pk", flat=True
)
# Pretranslate newly added entities for all locales
if new_entities and locales:
new_entities = list(set(new_entities))
pretranslate(db_project, locales=locales, entities=new_entities)
def sync_project(self,
project_pk,
sync_log_pk,
locale=None,
no_pull=False,
no_commit=False,
force=False):
"""Fetch the project with the given PK and perform sync on it."""
db_project = get_or_fail(
Project,
pk=project_pk,
message='Could not sync project with pk={0}, not found.'.format(
project_pk))
sync_log = get_or_fail(
SyncLog,
pk=sync_log_pk,
message=(
'Could not sync project {0}, log with pk={1} not found.'.format(
db_project.slug, sync_log_pk)))
log.info('Syncing project {0}.'.format(db_project.slug))
# Mark "now" at the start of sync to avoid messing with
# translations submitted during sync.
now = timezone.now()
project_sync_log = ProjectSyncLog.objects.create(sync_log=sync_log,
project=db_project,
start_time=now)
# We have to cache changed files before last revision is overriden by pull
if no_pull:
repos_changed = True # Assume changed.
else:
repos_changed, _ = pull_changes(db_project)
# If the repos haven't changed since the last sync and there are
# no Pontoon-side changes for this project, quit early.
if not force and not repos_changed and not db_project.needs_sync:
log.info('Skipping project {0}, no changes detected.'.format(
db_project.slug))
project_sync_log.skip()
return
# Do not sync resources if locale specified
if locale:
repo = locale.get_repository(db_project)
if repo:
sync_project_repo.delay(project_pk,
repo.pk,
project_sync_log.pk,
now,
locale=locale,
no_pull=no_pull,
no_commit=no_commit,
full_scan=force)
else:
try:
obsolete_vcs_entities, obsolete_vcs_resources, new_paths = perform_sync_project(
db_project, now, full_scan=force)
except MissingSourceDirectoryError, e:
log.error(e)
project_sync_log.skip()
return
log.info('Synced resources for project {0}.'.format(db_project.slug))
# No need to sync translations if it's a source repository
for repo in db_project.repositories.exclude(source_repo=True):
sync_project_repo.delay(project_pk,
repo.pk,
project_sync_log.pk,
now,
obsolete_vcs_entities,
obsolete_vcs_resources,
new_paths,
no_pull=no_pull,
no_commit=no_commit,
full_scan=force)
for repo in db_project.repositories.filter(source_repo=True):
repo.set_current_last_synced_revisions()
def sync_project_repo(self,
project_pk,
repo_pk,
project_sync_log_pk,
now,
obsolete_vcs_entities=None,
obsolete_vcs_resources=None,
new_paths=None,
locale=None,
no_pull=False,
no_commit=False,
full_scan=False):
db_project = get_or_fail(
Project,
pk=project_pk,
message='Could not sync project with pk={0}, not found.'.format(
project_pk))
repo = get_or_fail(
Repository,
pk=repo_pk,
message='Could not sync repo with pk={0}, not found.'.format(repo_pk))
project_sync_log = get_or_fail(
ProjectSyncLog,
pk=project_sync_log_pk,
message=(
'Could not sync project {0}, log with pk={1} not found.'.format(
db_project.slug, project_sync_log_pk)))
log.info('Syncing repo: {}'.format(repo.url))
repo_sync_log = RepositorySyncLog.objects.create(
project_sync_log=project_sync_log,
repository=repo,
start_time=timezone.now())
# Pull VCS changes in case we're on a different worker than the one
# sync started on.
if not no_pull:
pull_changes(db_project)
if locale:
locales = [locale]
else:
locales = repo.locales
# Cannot skip earlier - repo.locales is only available after pull_changes()
if not locales:
log.debug(
'Skipping repo `{0}` for project {1}, no locales to sync found within.'
.format(repo.url, db_project.slug))
repo_sync_log.end()
return
vcs_project = VCSProject(
db_project,
locales=locales,
obsolete_entities_paths=Resource.objects.obsolete_entities_paths(
obsolete_vcs_entities),
new_paths=new_paths,
full_scan=full_scan)
for locale in locales:
try:
with transaction.atomic():
# Skip locales that have nothing to sync
if vcs_project.synced_locales and locale not in vcs_project.synced_locales:
if obsolete_vcs_entities or obsolete_vcs_resources:
update_locale_project_locale_stats(locale, db_project)
continue
changeset = ChangeSet(db_project, vcs_project, now,
obsolete_vcs_entities,
obsolete_vcs_resources)
update_translations(db_project, vcs_project, locale, changeset)
changeset.execute()
update_translated_resources(db_project, vcs_project, changeset,
locale)
# Skip if none of the locales has anything to sync
# VCSProject.synced_locales is set on a first call to
# VCSProject.resources, which is set in
# pontoon.sync.core.update_translated_resources()
if len(vcs_project.synced_locales) == 0:
if obsolete_vcs_entities or obsolete_vcs_resources:
for l in locales:
update_locale_project_locale_stats(l, db_project)
db_project.aggregate_stats()
log.info(
'Skipping repo `{0}` for project {1}, none of the locales has anything to sync.'
.format(repo.url, db_project.slug))
repo_sync_log.end()
return
update_locale_project_locale_stats(locale, db_project)
# Clear out the "has_changed" markers now that we've finished
# syncing.
(ChangedEntityLocale.objects.filter(
entity__resource__project=db_project,
locale=locale,
when__lte=now).delete())
# Clean up any duplicate approvals at the end of sync right
# before we commit the transaction to avoid race conditions.
with connection.cursor() as cursor:
cursor.execute(
"""
UPDATE base_translation AS b
SET approved = FALSE, approved_date = NULL
WHERE
id IN
(SELECT trans.id FROM base_translation AS trans
LEFT JOIN base_entity AS ent ON ent.id = trans.entity_id
LEFT JOIN base_resource AS res ON res.id = ent.resource_id
WHERE locale_id = %(locale_id)s
AND res.project_id = %(project_id)s)
AND approved_date !=
(SELECT max(approved_date)
FROM base_translation
WHERE entity_id = b.entity_id
AND locale_id = b.locale_id
AND (plural_form = b.plural_form OR plural_form IS NULL));
""", {
'locale_id': locale.id,
'project_id': db_project.id
})
# Perform the commit last so that, if it succeeds, there is
# nothing after it to fail.
if not no_commit and locale in changeset.locales_to_commit:
commit_changes(db_project, vcs_project, changeset, locale)
except CommitToRepositoryException as err:
# Transaction aborted, log and move on to the next locale.
log.warning(
'Failed to sync locale {locale} for project {project} due to '
'commit error: {error}'.format(
locale=locale.code,
project=db_project.slug,
error=err,
))
with transaction.atomic():
db_project.aggregate_stats()
log.info('Synced translations for project {0} in locales {1}.'.format(
db_project.slug,
','.join(locale.code for locale in vcs_project.synced_locales)))
repo_sync_log.end()
def sync_translations(self,
project_pk,
project_sync_log_pk,
now,
project_changes=None,
obsolete_vcs_resources=None,
new_paths=None,
locale=None,
no_pull=False,
no_commit=False,
full_scan=False):
db_project = get_or_fail(
Project,
pk=project_pk,
message='Could not sync project with pk={0}, not found.'.format(
project_pk))
repos = db_project.translation_repositories()
repo_pk = repos[0].pk
repo = get_or_fail(
Repository,
pk=repo_pk,
message='Could not sync repo with pk={0}, not found.'.format(repo_pk))
project_sync_log = get_or_fail(
ProjectSyncLog,
pk=project_sync_log_pk,
message=(
'Could not sync project {0}, log with pk={1} not found.'.format(
db_project.slug, project_sync_log_pk)))
log.info('Syncing translations for project: {}'.format(db_project.slug))
repo_sync_log = RepositorySyncLog.objects.create(
project_sync_log=project_sync_log,
repository=repo,
start_time=timezone.now())
if locale:
locales = db_project.locales.filter(pk=locale.pk)
else:
locales = db_project.locales.all()
if not locales:
log.info(
'Skipping syncing translations for project {0}, no locales to sync found within.'
.format(db_project.slug))
repo_sync_log.end()
return
# If project repositories have API access, we can retrieve latest commit hashes and detect
# changed locales before the expensive VCS pull/clone operations. When performing full scan,
# we still need to sync all locales.
if not full_scan:
locales = get_changed_locales(db_project, locales, now)
# Pull VCS changes in case we're on a different worker than the one
# sync started on.
if not no_pull:
log.info('Pulling changes for project {0} started.'.format(
db_project.slug))
repos_changed, repo_locales = pull_changes(db_project, locales)
repos = repos.filter(pk__in=repo_locales.keys())
log.info('Pulling changes for project {0} complete.'.format(
db_project.slug))
changed_resources = []
obsolete_vcs_entities = []
if project_changes:
updated_entity_pks = []
for locale_code, db_entity, vcs_entity in project_changes['update_db']:
updated_entity_pks.append(db_entity.pk)
obsolete_entity_pks = project_changes['obsolete_db']
changed_resources = db_project.resources.filter(
Q(entities__date_created=now)
| Q(entities__pk__in=updated_entity_pks +
obsolete_entity_pks)).distinct()
obsolete_vcs_entities = project_changes['obsolete_db']
# If none of the repos has changed since the last sync and there are
# no Pontoon-side changes for this project, quit early.
if (not full_scan and not db_project.needs_sync and not repos_changed
and not (changed_resources or obsolete_vcs_resources)):
log.info('Skipping project {0}, no changes detected.'.format(
db_project.slug))
repo_sync_log.end()
return
obsolete_entities_paths = (
Resource.objects.obsolete_entities_paths(obsolete_vcs_entities)
if obsolete_vcs_entities else None)
vcs_project = VCSProject(db_project,
now,
locales=locales,
repo_locales=repo_locales,
obsolete_entities_paths=obsolete_entities_paths,
new_paths=new_paths,
full_scan=full_scan)
synced_locales = set()
failed_locales = set()
for locale in locales:
try:
with transaction.atomic():
# Sets VCSProject.synced_locales, needed to skip early
if not vcs_project.synced_locales:
vcs_project.resources
# Skip all locales if none of the them has anything to sync
if len(vcs_project.synced_locales) == 0:
break
# Skip locales that have nothing to sync
if vcs_project.synced_locales and locale not in vcs_project.synced_locales:
continue
changeset = ChangeSet(db_project, vcs_project, now, locale)
update_translations(db_project, vcs_project, locale, changeset)
changeset.execute()
update_translated_resources(db_project, vcs_project, locale)
update_locale_project_locale_stats(locale, db_project)
# Clear out the "has_changed" markers now that we've finished
# syncing.
(ChangedEntityLocale.objects.filter(
entity__resource__project=db_project,
locale=locale,
when__lte=now).delete())
# Clean up any duplicate approvals at the end of sync right
# before we commit the transaction to avoid race conditions.
with connection.cursor() as cursor:
cursor.execute(
"""
UPDATE base_translation AS b
SET approved = FALSE, approved_date = NULL
WHERE
id IN
(SELECT trans.id FROM base_translation AS trans
LEFT JOIN base_entity AS ent ON ent.id = trans.entity_id
LEFT JOIN base_resource AS res ON res.id = ent.resource_id
WHERE locale_id = %(locale_id)s
AND res.project_id = %(project_id)s)
AND approved_date !=
(SELECT max(approved_date)
FROM base_translation
WHERE entity_id = b.entity_id
AND locale_id = b.locale_id
AND (plural_form = b.plural_form OR plural_form IS NULL));
""", {
'locale_id': locale.id,
'project_id': db_project.id
})
# Perform the commit last so that, if it succeeds, there is
# nothing after it to fail.
if not no_commit and locale in changeset.locales_to_commit:
commit_changes(db_project, vcs_project, changeset, locale)
log.info(
'Synced locale {locale} for project {project}.'.format(
locale=locale.code,
project=db_project.slug,
))
synced_locales.add(locale.code)
except CommitToRepositoryException as err:
# Transaction aborted, log and move on to the next locale.
log.warning(
'Failed to sync locale {locale} for project {project} due to '
'commit error: {error}'.format(
locale=locale.code,
project=db_project.slug,
error=err,
))
failed_locales.add(locale.code)
# If sources have changed, update stats for all locales.
if changed_resources or obsolete_vcs_resources:
for locale in db_project.locales.all():
# Already synced.
if locale.code in synced_locales:
continue
# We have files: update all translated resources.
if locale in locales:
update_translated_resources(db_project, vcs_project, locale)
# We don't have files: we can still update asymmetric translated resources.
else:
update_translated_resources_no_files(db_project, locale,
changed_resources)
update_locale_project_locale_stats(locale, db_project)
synced_locales.add(locale.code)
log.info(
'Synced source changes for locale {locale} for project {project}.'
.format(
locale=locale.code,
project=db_project.slug,
))
db_project.aggregate_stats()
if synced_locales:
log.info('Synced translations for project {0} in locales {1}.'.format(
db_project.slug, ','.join(synced_locales)))
elif failed_locales:
log.info(
'Failed to sync translations for project {0} due to commit error.'.
format(db_project.slug))
else:
log.info(
'Skipping syncing translations for project {0}, none of the locales '
'has anything to sync.'.format(db_project.slug))
for r in repos:
r.set_last_synced_revisions(locales=repo_locales[r.pk].exclude(
code__in=failed_locales))
repo_sync_log.end()
def sync_translations(self, project_pk, repo_pk, project_sync_log_pk, now, project_changes=None,
obsolete_vcs_resources=None, new_paths=None, locale=None, no_pull=False, no_commit=False,
full_scan=False):
db_project = get_or_fail(Project, pk=project_pk,
message='Could not sync project with pk={0}, not found.'.format(project_pk))
repo = get_or_fail(Repository, pk=repo_pk,
message='Could not sync repo with pk={0}, not found.'.format(repo_pk))
project_sync_log = get_or_fail(ProjectSyncLog, pk=project_sync_log_pk,
message=('Could not sync project {0}, log with pk={1} not found.'
.format(db_project.slug, project_sync_log_pk)))
log.info('Syncing translations for repo: {}'.format(repo.url))
repo_sync_log = RepositorySyncLog.objects.create(
project_sync_log=project_sync_log,
repository=repo,
start_time=timezone.now()
)
if locale:
locales = [locale]
else:
locales = repo.locales
if not locales:
log.info('Skipping repo `{0}` for project {1}, no locales to sync found within.'
.format(repo.url, db_project.slug))
repo_sync_log.end()
return
# Pull VCS changes in case we're on a different worker than the one
# sync started on.
if not no_pull:
repos_changed = pull_changes(db_project)
resources_changed = []
obsolete_vcs_entities = []
if project_changes:
resources_changed = (
project_changes['update_db'] +
project_changes['obsolete_db'] +
project_changes['create_db']
)
obsolete_vcs_entities = project_changes['obsolete_db']
# If none of the repos has changed since the last sync and there are
# no Pontoon-side changes for this project, quit early.
if not full_scan and not db_project.needs_sync and not repos_changed and not (resources_changed or obsolete_vcs_resources):
log.info('Skipping project {0}, no changes detected.'.format(db_project.slug))
repo_sync_log.end()
return
obsolete_entities_paths = Resource.objects.obsolete_entities_paths(obsolete_vcs_entities) if obsolete_vcs_entities else None
vcs_project = VCSProject(
db_project,
locales=locales,
obsolete_entities_paths=obsolete_entities_paths,
new_paths=new_paths,
full_scan=full_scan
)
failed_locales = set()
for locale in locales:
try:
with transaction.atomic():
# Sets VCSProject.synced_locales, needed to skip early
if not vcs_project.synced_locales:
vcs_project.resources
if not obsolete_vcs_resources:
# Skip all locales if none of the them has anything to sync
if len(vcs_project.synced_locales) == 0:
if resources_changed:
for l in locales:
update_translated_resources(db_project, vcs_project, l)
update_locale_project_locale_stats(l, db_project)
db_project.aggregate_stats()
log.info('Skipping repo `{0}` for project {1}, none of the locales has anything to sync.'
.format(repo.url, db_project.slug))
repo.set_last_synced_revisions()
repo_sync_log.end()
return
# Skip locales that have nothing to sync
if vcs_project.synced_locales and locale not in vcs_project.synced_locales:
if resources_changed:
update_translated_resources(db_project, vcs_project, locale)
update_locale_project_locale_stats(locale, db_project)
log.debug('Skipping locale `{0}` for project {1}, no changes detected.'
.format(locale.code, db_project.slug))
continue
changeset = ChangeSet(db_project, vcs_project, now, obsolete_vcs_entities, obsolete_vcs_resources, locale)
update_translations(db_project, vcs_project, locale, changeset)
changeset.execute()
update_translated_resources(db_project, vcs_project, locale)
update_locale_project_locale_stats(locale, db_project)
# Clear out the "has_changed" markers now that we've finished
# syncing.
(ChangedEntityLocale.objects
.filter(entity__resource__project=db_project,
locale=locale,
when__lte=now)
.delete())
# Clean up any duplicate approvals at the end of sync right
# before we commit the transaction to avoid race conditions.
with connection.cursor() as cursor:
cursor.execute("""
UPDATE base_translation AS b
SET approved = FALSE, approved_date = NULL
WHERE
id IN
(SELECT trans.id FROM base_translation AS trans
LEFT JOIN base_entity AS ent ON ent.id = trans.entity_id
LEFT JOIN base_resource AS res ON res.id = ent.resource_id
WHERE locale_id = %(locale_id)s
AND res.project_id = %(project_id)s)
AND approved_date !=
(SELECT max(approved_date)
FROM base_translation
WHERE entity_id = b.entity_id
AND locale_id = b.locale_id
AND (plural_form = b.plural_form OR plural_form IS NULL));
""", {
'locale_id': locale.id,
'project_id': db_project.id
})
# Perform the commit last so that, if it succeeds, there is
# nothing after it to fail.
if not no_commit and locale in changeset.locales_to_commit:
commit_changes(db_project, vcs_project, changeset, locale)
log.info(
'Synced locale {locale} for project {project}.'.format(
locale=locale.code,
project=db_project.slug,
)
)
except CommitToRepositoryException as err:
# Transaction aborted, log and move on to the next locale.
log.warning(
'Failed to sync locale {locale} for project {project} due to '
'commit error: {error}'.format(
locale=locale.code,
project=db_project.slug,
error=err,
)
)
failed_locales.add(locale)
with transaction.atomic():
db_project.aggregate_stats()
synced_locales = [locale.code for locale in (vcs_project.synced_locales - failed_locales)]
if synced_locales:
log.info('Synced translations for project {0} in locales {1}.'.format(
db_project.slug, ','.join(synced_locales)
))
else:
log.info('Failed to sync translations for project {0} due to commit error.'.format(
db_project.slug
))
repo.set_last_synced_revisions(exclude=failed_locales)
repo_sync_log.end()
def sync_translations(
self, project_pk, project_sync_log_pk, now, added_paths=None, removed_paths=None,
changed_paths=None, locale=None, no_pull=False, no_commit=False, full_scan=False,
):
db_project = get_or_fail(
Project,
pk=project_pk,
message='Could not sync project with pk={0}, not found.'.format(project_pk)
)
repos = db_project.translation_repositories()
repo_pk = repos[0].pk
repo = get_or_fail(
Repository,
pk=repo_pk,
message='Could not sync repo with pk={0}, not found.'.format(repo_pk)
)
project_sync_log = get_or_fail(
ProjectSyncLog,
pk=project_sync_log_pk,
message=(
'Could not sync project {0}, log with pk={1} not found.'
.format(db_project.slug, project_sync_log_pk)
)
)
log.info('Syncing translations for project: {}'.format(db_project.slug))
repo_sync_log = RepositorySyncLog.objects.create(
project_sync_log=project_sync_log,
repository=repo,
start_time=timezone.now()
)
if locale:
locales = db_project.locales.filter(pk=locale.pk)
else:
locales = db_project.locales.all()
if not locales:
log.info('Skipping syncing translations for project {0}, no locales to sync found within.'
.format(db_project.slug))
repo_sync_log.end()
return
# If project repositories have API access, we can retrieve latest commit hashes and detect
# changed locales before the expensive VCS pull/clone operations. When performing full scan,
# we still need to sync all locales.
if not full_scan:
locales = get_changed_locales(db_project, locales, now)
readonly_locales = db_project.locales.filter(project_locale__readonly=True)
added_and_changed_resources = db_project.resources.filter(
path__in=list(added_paths or []) + list(changed_paths or [])
).distinct()
# We should also sync files for which source file change - but only for read-only locales.
# See bug 1372151 for more details.
if added_and_changed_resources:
changed_locales_pks = [l.pk for l in locales]
readonly_locales_pks = [l.pk for l in readonly_locales]
locales = db_project.locales.filter(
pk__in=changed_locales_pks + readonly_locales_pks
)
# Pull VCS changes in case we're on a different worker than the one
# sync started on.
if not no_pull:
log.info('Pulling changes for project {0} started.'.format(db_project.slug))
repos_changed, repo_locales = pull_changes(db_project, locales)
repos = repos.filter(pk__in=repo_locales.keys())
log.info('Pulling changes for project {0} complete.'.format(db_project.slug))
# If none of the repos has changed since the last sync and there are
# no Pontoon-side changes for this project, quit early.
if (
not full_scan and
not db_project.needs_sync and
not repos_changed and
not (added_paths or removed_paths or changed_paths)
):
log.info('Skipping project {0}, no changes detected.'.format(db_project.slug))
repo_sync_log.end()
return
vcs_project = VCSProject(
db_project,
now,
locales=locales,
repo_locales=repo_locales,
added_paths=added_paths,
changed_paths=changed_paths,
full_scan=full_scan
)
synced_locales = set()
failed_locales = set()
for locale in locales:
try:
with transaction.atomic():
# Sets VCSProject.synced_locales, needed to skip early
if not vcs_project.synced_locales:
vcs_project.resources
# Skip all locales if none of the them has anything to sync
if len(vcs_project.synced_locales) == 0:
break
# Skip locales that have nothing to sync
if vcs_project.synced_locales and locale not in vcs_project.synced_locales:
continue
changeset = ChangeSet(db_project, vcs_project, now, locale)
update_translations(db_project, vcs_project, locale, changeset)
changeset.execute()
update_translated_resources(db_project, vcs_project, locale)
update_locale_project_locale_stats(locale, db_project)
# Clear out the "has_changed" markers now that we've finished
# syncing.
(ChangedEntityLocale.objects
.filter(entity__resource__project=db_project,
locale=locale,
when__lte=now)
.delete())
# Perform the commit last so that, if it succeeds, there is
# nothing after it to fail.
if (
not no_commit and
locale in changeset.locales_to_commit and
locale not in readonly_locales
):
commit_changes(db_project, vcs_project, changeset, locale)
log.info(
'Synced locale {locale} for project {project}.'.format(
locale=locale.code,
project=db_project.slug,
)
)
synced_locales.add(locale.code)
except CommitToRepositoryException as err:
# Transaction aborted, log and move on to the next locale.
log.warning(
'Failed to sync locale {locale} for project {project} due to '
'commit error: {error}'.format(
locale=locale.code,
project=db_project.slug,
error=err,
)
)
failed_locales.add(locale.code)
# If sources have changed, update stats for all locales.
if added_paths or removed_paths or changed_paths:
for locale in db_project.locales.all():
# Already synced.
if locale.code in synced_locales:
continue
# We have files: update all translated resources.
if locale in locales:
update_translated_resources(db_project, vcs_project, locale)
# We don't have files: we can still update asymmetric translated resources.
else:
update_translated_resources_no_files(
db_project,
locale,
added_and_changed_resources,
)
update_locale_project_locale_stats(locale, db_project)
synced_locales.add(locale.code)
log.info(
'Synced source changes for locale {locale} for project {project}.'.format(
locale=locale.code,
project=db_project.slug,
)
)
db_project.aggregate_stats()
if synced_locales:
log.info('Synced translations for project {0} in locales {1}.'.format(
db_project.slug, ','.join(synced_locales)
))
elif failed_locales:
log.info('Failed to sync translations for project {0} due to commit error.'.format(
db_project.slug
))
else:
log.info(
'Skipping syncing translations for project {0}, none of the locales '
'has anything to sync.'.format(db_project.slug)
)
for r in repos:
r.set_last_synced_revisions(
locales=repo_locales[r.pk].exclude(code__in=failed_locales)
)
repo_sync_log.end()
def sync_project_repo(self, project_pk, repo_pk, project_sync_log_pk, now, obsolete_vcs_entities=None,
obsolete_vcs_resources=None, locale=None, no_pull=False, no_commit=False,
full_scan=False):
db_project = get_or_fail(Project, pk=project_pk,
message='Could not sync project with pk={0}, not found.'.format(project_pk))
repo = get_or_fail(Repository, pk=repo_pk,
message='Could not sync repo with pk={0}, not found.'.format(repo_pk))
project_sync_log = get_or_fail(ProjectSyncLog, pk=project_sync_log_pk,
message=('Could not sync project {0}, log with pk={1} not found.'
.format(db_project.slug, project_sync_log_pk)))
log.info('Syncing repo: {}'.format(repo.url))
repo_sync_log = RepositorySyncLog.objects.create(
project_sync_log=project_sync_log,
repository=repo,
start_time=timezone.now()
)
# Pull VCS changes in case we're on a different worker than the one
# sync started on.
if not no_pull:
pull_changes(db_project)
if locale:
locales = [locale]
else:
locales = repo.locales
vcs_project = VCSProject(
db_project,
locales=locales,
obsolete_entities_paths=Resource.objects.obsolete_entities_paths(obsolete_vcs_entities),
full_scan=full_scan
)
for locale in locales:
try:
with transaction.atomic():
# Skip locales that have nothing to sync
if vcs_project.synced_locales and locale not in vcs_project.synced_locales:
continue
changeset = ChangeSet(db_project, vcs_project, now, obsolete_vcs_entities, obsolete_vcs_resources)
update_translations(db_project, vcs_project, locale, changeset)
changeset.execute()
update_translated_resources(db_project, vcs_project, changeset, locale)
# Skip if none of the locales has anything to sync
# VCSProject.synced_locales is set on a first call to
# VCSProject.resources, which is set in
# pontoon.sync.core.update_translated_resources()
if len(vcs_project.synced_locales) == 0:
log.info('Skipping repo `{0}` for project {1}, none of the locales has anything to sync.'
.format(repo.url, db_project.slug))
end_repo_sync(repo, repo_sync_log)
return
locale.aggregate_stats()
locale.project_locale.get(project=db_project).aggregate_stats()
# Clear out the "has_changed" markers now that we've finished
# syncing.
(ChangedEntityLocale.objects
.filter(entity__resource__project=db_project,
locale=locale,
when__lte=now)
.delete())
db_project.has_changed = False
db_project.save(update_fields=['has_changed'])
# Clean up any duplicate approvals at the end of sync right
# before we commit the transaction to avoid race conditions.
with connection.cursor() as cursor:
cursor.execute("""
UPDATE base_translation AS b
SET approved = FALSE, approved_date = NULL
WHERE
id IN
(SELECT trans.id FROM base_translation AS trans
LEFT JOIN base_entity AS ent ON ent.id = trans.entity_id
LEFT JOIN base_resource AS res ON res.id = ent.resource_id
WHERE locale_id = %(locale_id)s
AND res.project_id = %(project_id)s)
AND approved_date !=
(SELECT max(approved_date)
FROM base_translation
WHERE entity_id = b.entity_id
AND locale_id = b.locale_id
AND (plural_form = b.plural_form OR plural_form IS NULL));
""", {
'locale_id': locale.id,
'project_id': db_project.id
})
# Perform the commit last so that, if it succeeds, there is
# nothing after it to fail.
if not no_commit and locale in changeset.locales_to_commit:
commit_changes(db_project, vcs_project, changeset, locale)
except CommitToRepositoryException as err:
# Transaction aborted, log and move on to the next locale.
log.warning(
'Failed to sync locale {locale} for project {project} due to '
'commit error: {error}'.format(
locale=locale.code,
project=db_project.slug,
error=err,
)
)
with transaction.atomic():
db_project.aggregate_stats()
log.info('Synced translations for project {0} in locales {1}.'.format(
db_project.slug, ','.join(locale.code for locale in vcs_project.synced_locales)
))
end_repo_sync(repo, repo_sync_log)
def sync_project_repo(self, project_pk, repo_pk, project_sync_log_pk, now,
no_pull=False, no_commit=False):
db_project = get_or_fail(Project, pk=project_pk,
message='Could not sync project with pk={0}, not found.'.format(project_pk))
repo = get_or_fail(Repository, pk=repo_pk,
message='Could not sync repo with pk={0}, not found.'.format(project_pk))
project_sync_log = get_or_fail(ProjectSyncLog, pk=project_sync_log_pk,
message=('Could not sync project {0}, log with pk={1} not found.'
.format(db_project.slug, project_sync_log_pk)))
repo_sync_log = RepositorySyncLog.objects.create(
project_sync_log=project_sync_log,
repository=repo,
start_time=timezone.now()
)
# Pull VCS changes in case we're on a different worker than the one
# sync started on.
if not no_pull:
pull_changes(db_project)
if len(repo.locales) < 1:
log.warning('Could not sync repo `{0}`, no locales found within.'
.format(repo.url))
repo_sync_log.end_time = timezone.now()
repo_sync_log.save(update_fields=['end_time'])
return
vcs_project = VCSProject(db_project, locales=repo.locales)
for locale in repo.locales:
try:
with transaction.atomic():
changeset = ChangeSet(db_project, vcs_project, now)
update_translations(db_project, vcs_project, locale, changeset)
changeset.execute()
update_project_stats(db_project, vcs_project, changeset, locale)
# Clear out the "has_changed" markers now that we've finished
# syncing.
(ChangedEntityLocale.objects
.filter(entity__resource__project=db_project,
locale=locale,
when__lte=now)
.delete())
db_project.has_changed = False
db_project.save(update_fields=['has_changed'])
# Clean up any duplicate approvals at the end of sync right
# before we commit the transaction to avoid race conditions.
with connection.cursor() as cursor:
cursor.execute("""
UPDATE base_translation AS b
SET approved = FALSE, approved_date = NULL
WHERE
id IN
(SELECT trans.id FROM base_translation AS trans
LEFT JOIN base_entity AS ent ON ent.id = trans.entity_id
LEFT JOIN base_resource AS res ON res.id = ent.resource_id
WHERE locale_id = %(locale_id)s
AND res.project_id = %(project_id)s)
AND approved_date !=
(SELECT max(approved_date)
FROM base_translation
WHERE entity_id = b.entity_id
AND locale_id = b.locale_id
AND (plural_form = b.plural_form OR plural_form IS NULL));
""", {
'locale_id': locale.id,
'project_id': db_project.id
})
# Perform the commit last so that, if it succeeds, there is
# nothing after it to fail.
if not no_commit and locale in changeset.locales_to_commit:
commit_changes(db_project, vcs_project, changeset, locale)
except CommitToRepositoryException as err:
# Transaction aborted, log and move on to the next locale.
log.warning(
'Failed to sync locale {locale} for project {project} due to '
'commit error: {error}'.format(
locale=locale.code,
project=db_project.slug,
error=err,
)
)
repo_sync_log.end_time = timezone.now()
repo_sync_log.save()
log.info('Synced translations for project {0} in locales {1}.'.format(
db_project.slug, ','.join(locale.code for locale in repo.locales)
))
def sync_project(self, project_pk, sync_log_pk, locale=None, no_pull=False, no_commit=False, force=False):
"""Fetch the project with the given PK and perform sync on it."""
db_project = get_or_fail(Project, pk=project_pk,
message='Could not sync project with pk={0}, not found.'.format(project_pk))
sync_log = get_or_fail(SyncLog, pk=sync_log_pk,
message=('Could not sync project {0}, log with pk={1} not found.'
.format(db_project.slug, sync_log_pk)))
log.info('Syncing project {0}.'.format(db_project.slug))
# Mark "now" at the start of sync to avoid messing with
# translations submitted during sync.
now = timezone.now()
project_sync_log = ProjectSyncLog.objects.create(
sync_log=sync_log,
project=db_project,
start_time=now
)
# We have to cache changed files before last revision is overriden by pull
if no_pull:
repos_changed = True # Assume changed.
else:
repos_changed, _ = pull_changes(db_project)
# If the repos haven't changed since the last sync and there are
# no Pontoon-side changes for this project, quit early.
if not force and not repos_changed and not db_project.needs_sync:
log.info('Skipping project {0}, no changes detected.'.format(db_project.slug))
project_sync_log.skip()
return
# Do not sync resources if locale specified
if locale:
repo = locale.get_repository(db_project)
if repo:
sync_project_repo.delay(
project_pk,
repo.pk,
project_sync_log.pk,
now,
locale=locale,
no_pull=no_pull,
no_commit=no_commit,
full_scan=force
)
else:
try:
obsolete_vcs_entities, obsolete_vcs_resources = perform_sync_project(db_project, now, full_scan=force)
except MissingSourceDirectoryError, e:
log.error(e)
project_sync_log.skip()
return
log.info('Synced resources for project {0}.'.format(db_project.slug))
# No need to sync translations if it's a source repository
for repo in db_project.repositories.exclude(source_repo=True):
sync_project_repo.delay(
project_pk,
repo.pk,
project_sync_log.pk,
now,
obsolete_vcs_entities,
obsolete_vcs_resources,
no_pull=no_pull,
no_commit=no_commit,
full_scan=force
)
for repo in db_project.repositories.filter(source_repo=True):
repo.set_current_last_synced_revisions()
def sync_translations(
self, project_pk, project_sync_log_pk, now, project_changes=None, obsolete_vcs_resources=None,
new_paths=None, locale=None, no_pull=False, no_commit=False, full_scan=False
):
db_project = get_or_fail(
Project,
pk=project_pk,
message='Could not sync project with pk={0}, not found.'.format(project_pk)
)
repos = db_project.translation_repositories()
repo_pk = repos[0].pk
repo = get_or_fail(
Repository,
pk=repo_pk,
message='Could not sync repo with pk={0}, not found.'.format(repo_pk)
)
project_sync_log = get_or_fail(
ProjectSyncLog,
pk=project_sync_log_pk,
message=(
'Could not sync project {0}, log with pk={1} not found.'
.format(db_project.slug, project_sync_log_pk)
)
)
log.info('Syncing translations for project: {}'.format(db_project.slug))
repo_sync_log = RepositorySyncLog.objects.create(
project_sync_log=project_sync_log,
repository=repo,
start_time=timezone.now()
)
if locale:
locales = db_project.locales.filter(pk=locale.pk)
else:
locales = db_project.locales.all()
if not locales:
log.info('Skipping syncing translations for project {0}, no locales to sync found within.'
.format(db_project.slug))
repo_sync_log.end()
return
# If project repositories have API access, we can retrieve latest commit hashes and detect
# changed locales before the expensive VCS pull/clone operations. When performing full scan,
# we still need to sync all locales.
if not full_scan:
locales = get_changed_locales(db_project, locales, now)
# Pull VCS changes in case we're on a different worker than the one
# sync started on.
if not no_pull:
log.info('Pulling changes for project {0} started.'.format(db_project.slug))
repos_changed, repo_locales = pull_changes(db_project, locales)
repos = repos.filter(pk__in=repo_locales.keys())
log.info('Pulling changes for project {0} complete.'.format(db_project.slug))
changed_resources = []
obsolete_vcs_entities = []
if project_changes:
updated_entity_pks = []
for locale_code, db_entity, vcs_entity in project_changes['update_db']:
updated_entity_pks.append(db_entity.pk)
obsolete_entity_pks = project_changes['obsolete_db']
changed_resources = db_project.resources.filter(
Q(entities__date_created=now) |
Q(entities__pk__in=updated_entity_pks + obsolete_entity_pks)
).distinct()
obsolete_vcs_entities = project_changes['obsolete_db']
# If none of the repos has changed since the last sync and there are
# no Pontoon-side changes for this project, quit early.
if (
not full_scan and
not db_project.needs_sync and
not repos_changed and
not (changed_resources or obsolete_vcs_resources)
):
log.info('Skipping project {0}, no changes detected.'.format(db_project.slug))
repo_sync_log.end()
return
obsolete_entities_paths = (
Resource.objects.obsolete_entities_paths(obsolete_vcs_entities) if obsolete_vcs_entities
else None
)
vcs_project = VCSProject(
db_project,
now,
locales=locales,
repo_locales=repo_locales,
obsolete_entities_paths=obsolete_entities_paths,
new_paths=new_paths,
full_scan=full_scan
)
synced_locales = set()
failed_locales = set()
for locale in locales:
try:
with transaction.atomic():
# Sets VCSProject.synced_locales, needed to skip early
if not vcs_project.synced_locales:
vcs_project.resources
# Skip all locales if none of the them has anything to sync
if len(vcs_project.synced_locales) == 0:
break
# Skip locales that have nothing to sync
if vcs_project.synced_locales and locale not in vcs_project.synced_locales:
continue
changeset = ChangeSet(db_project, vcs_project, now, locale)
update_translations(db_project, vcs_project, locale, changeset)
changeset.execute()
update_translated_resources(db_project, vcs_project, locale)
update_locale_project_locale_stats(locale, db_project)
# Clear out the "has_changed" markers now that we've finished
# syncing.
(ChangedEntityLocale.objects
.filter(entity__resource__project=db_project,
locale=locale,
when__lte=now)
.delete())
# Clean up any duplicate approvals at the end of sync right
# before we commit the transaction to avoid race conditions.
with connection.cursor() as cursor:
cursor.execute("""
UPDATE base_translation AS b
SET approved = FALSE, approved_date = NULL
WHERE
id IN
(SELECT trans.id FROM base_translation AS trans
LEFT JOIN base_entity AS ent ON ent.id = trans.entity_id
LEFT JOIN base_resource AS res ON res.id = ent.resource_id
WHERE locale_id = %(locale_id)s
AND res.project_id = %(project_id)s)
AND approved_date !=
(SELECT max(approved_date)
FROM base_translation
WHERE entity_id = b.entity_id
AND locale_id = b.locale_id
AND (plural_form = b.plural_form OR plural_form IS NULL));
""", {
'locale_id': locale.id,
'project_id': db_project.id
})
# Perform the commit last so that, if it succeeds, there is
# nothing after it to fail.
if not no_commit and locale in changeset.locales_to_commit:
commit_changes(db_project, vcs_project, changeset, locale)
log.info(
'Synced locale {locale} for project {project}.'.format(
locale=locale.code,
project=db_project.slug,
)
)
synced_locales.add(locale.code)
except CommitToRepositoryException as err:
# Transaction aborted, log and move on to the next locale.
log.warning(
'Failed to sync locale {locale} for project {project} due to '
'commit error: {error}'.format(
locale=locale.code,
project=db_project.slug,
error=err,
)
)
failed_locales.add(locale.code)
# If sources have changed, update stats for all locales.
if changed_resources or obsolete_vcs_resources:
for locale in db_project.locales.all():
# Already synced.
if locale.code in synced_locales:
continue
# We have files: update all translated resources.
if locale in locales:
update_translated_resources(db_project, vcs_project, locale)
# We don't have files: we can still update asymmetric translated resources.
else:
update_translated_resources_no_files(db_project, locale, changed_resources)
update_locale_project_locale_stats(locale, db_project)
synced_locales.add(locale.code)
log.info(
'Synced source changes for locale {locale} for project {project}.'.format(
locale=locale.code,
project=db_project.slug,
)
)
db_project.aggregate_stats()
if synced_locales:
log.info('Synced translations for project {0} in locales {1}.'.format(
db_project.slug, ','.join(synced_locales)
))
elif failed_locales:
log.info('Failed to sync translations for project {0} due to commit error.'.format(
db_project.slug
))
else:
log.info(
'Skipping syncing translations for project {0}, none of the locales '
'has anything to sync.'.format(db_project.slug)
)
for r in repos:
r.set_last_synced_revisions(
locales=repo_locales[r.pk].exclude(code__in=failed_locales)
)
repo_sync_log.end()
