def setUp(self):
# Force the checkout path to point to a test directory to make
# resource file loading pass during tests.
checkout_path_patch = patch.object(Project,
'checkout_path',
new_callable=PropertyMock,
return_value=os.path.join(
TEST_CHECKOUT_PATH,
'no_resources_test'))
self.mock_checkout_path = checkout_path_patch.start()
self.addCleanup(checkout_path_patch.stop)
self.project = ProjectFactory.create()
self.vcs_project = VCSProject(self.project)
def setUp(self):
self.locale, _ = Locale.objects.get_or_create(code='fr')
self.repository = RepositoryFactory()
self.db_project = ProjectFactory.create(
repositories=[self.repository],
)
checkout_path_patch = patch.object(
Repository,
'checkout_path',
new_callable=PropertyMock,
return_value=PROJECT_CONFIG_CHECKOUT_PATH
)
self.mock_checkout_path = checkout_path_patch.start()
self.addCleanup(checkout_path_patch.stop)
self.resource_strings = ResourceFactory.create(
project=self.db_project,
path='values/strings.properties',
)
self.resource_strings_reality = ResourceFactory.create(
project=self.db_project,
path='values/strings_reality.properties',
)
# Make sure VCSConfiguration instance is initialized
self.db_project.configuration_file = 'l10n.toml'
self.vcs_project = VCSProject(self.db_project)
self.vcs_project.configuration.configuration_path = os.path.join(
PROJECT_CONFIG_CHECKOUT_PATH,
self.db_project.configuration_file,
)
def sync_project(db_project, now):
# Only load source resources for updating entities.
vcs_project = VCSProject(db_project, locales=[])
with transaction.atomic():
update_resources(db_project, vcs_project)
changeset = ChangeSet(db_project, vcs_project, now)
update_entities(db_project, vcs_project, changeset)
changeset.execute()
def sync_project(db_project, now, full_scan=False):
vcs_project = VCSProject(db_project, locales=[], full_scan=full_scan)
with transaction.atomic():
removed_paths = update_resources(db_project, vcs_project)
changeset = ChangeSet(db_project, vcs_project, now)
update_entities(db_project, vcs_project, changeset)
changeset.execute()
return changeset.changes['obsolete_db'], removed_paths
def update_originals(db_project, now, full_scan=False):
vcs_project = VCSProject(db_project, locales=[], full_scan=full_scan)
with transaction.atomic():
added_paths, removed_paths, changed_paths = update_resources(db_project, vcs_project)
changeset = ChangeSet(db_project, vcs_project, now)
update_entities(db_project, vcs_project, changeset)
changeset.execute()
return added_paths, removed_paths, changed_paths
def setUp(self):
# Force the checkout path to point to a test directory to make
# resource file loading pass during tests.
checkout_path_patch = patch.object(
Project,
'checkout_path',
new_callable=PropertyMock,
return_value=os.path.join(TEST_CHECKOUT_PATH, 'no_resources_test')
)
self.mock_checkout_path = checkout_path_patch.start()
self.addCleanup(checkout_path_patch.stop)
self.project = ProjectFactory.create()
self.vcs_project = VCSProject(self.project)
def setUp(self):
self.locale, _ = Locale.objects.get_or_create(code='fr')
self.repository = RepositoryFactory()
self.db_project = ProjectFactory.create(repositories=[self.repository
], )
self.resource_strings = ResourceFactory.create(
project=self.db_project,
path='strings.properties',
)
self.resource_strings_reality = ResourceFactory.create(
project=self.db_project,
path='strings_reality.properties',
)
# Make sure VCSConfiguration instance is initialized
self.db_project.configuration_file = 'l10n.toml'
self.vcs_project = VCSProject(self.db_project)
class VCSProjectTests(TestCase):
def setUp(self):
# Force the checkout path to point to a test directory to make
# resource file loading pass during tests.
checkout_path_patch = patch.object(
Project,
'checkout_path',
new_callable=PropertyMock,
return_value=os.path.join(TEST_CHECKOUT_PATH, 'no_resources_test')
)
self.mock_checkout_path = checkout_path_patch.start()
self.addCleanup(checkout_path_patch.stop)
self.project = ProjectFactory.create()
self.vcs_project = VCSProject(self.project)
def test_relative_resource_paths(self):
self.vcs_project.source_directory_path = Mock(return_value='/root/')
self.vcs_project.resources_for_path = Mock(return_value=[
'/root/foo.po',
'/root/meh/bar.po'
])
assert_equal(
list(self.vcs_project.relative_resource_paths()),
['foo.po', 'meh/bar.po']
)
def test_relative_resource_paths_pot(self):
"""
If a resource ends in .pot, replace the extension with .po since
relative paths are used within non-source locales that do not
have .pot files.
"""
self.vcs_project.source_directory_path = Mock(return_value='/root/')
self.vcs_project.resources_for_path = Mock(return_value=[
'/root/foo.pot',
'/root/meh/bar.pot'
])
assert_equal(
list(self.vcs_project.relative_resource_paths()),
['foo.po', 'meh/bar.po']
)
def test_source_directory_path_no_resource(self):
"""
When searching for source directories, do not match directories that
do not contain resource files.
"""
checkout_path = os.path.join(TEST_CHECKOUT_PATH, 'no_resources_test')
self.mock_checkout_path.return_value = checkout_path
assert_equal(
self.vcs_project.source_directory_path(),
os.path.join(checkout_path, 'real_resources', 'templates')
)
def test_source_directory_scoring_templates(self):
"""
When searching for source directories, prefer directories named
`templates` over all others.
"""
checkout_path = os.path.join(TEST_CHECKOUT_PATH, 'scoring_templates_test')
self.mock_checkout_path.return_value = checkout_path
assert_equal(
self.vcs_project.source_directory_path(),
os.path.join(checkout_path, 'templates')
)
def test_source_directory_scoring_en_US(self):
"""
When searching for source directories, prefer directories named
`en-US` over others besides `templates`.
"""
checkout_path = os.path.join(TEST_CHECKOUT_PATH, 'scoring_en_US_test')
self.mock_checkout_path.return_value = checkout_path
assert_equal(
self.vcs_project.source_directory_path(),
os.path.join(checkout_path, 'en-US')
)
def test_source_directory_scoring_source_files(self):
"""
When searching for source directories, prefer directories with
source-only formats over all others.
"""
checkout_path = os.path.join(TEST_CHECKOUT_PATH, 'scoring_source_files_test')
self.mock_checkout_path.return_value = checkout_path
assert_equal(
self.vcs_project.source_directory_path(),
os.path.join(checkout_path, 'en') # en has pot files in it
)
def test_resources_parse_error(self):
"""
If VCSResource() raises a ParseError while loading, log an error
and skip the resource.
"""
self.vcs_project.relative_resource_paths = Mock(return_value=['failure', 'success'])
# Fail only if the path is failure so we can test the ignore.
def vcs_resource_constructor(project, path, locales=None):
if path == 'failure':
raise ParseError('error message')
else:
return 'successful resource'
with patch('pontoon.sync.vcs.models.VCSResource') as MockVCSResource, \
patch('pontoon.sync.vcs.models.log') as mock_log:
MockVCSResource.side_effect = vcs_resource_constructor
assert_equal(self.vcs_project.resources, {'success': 'successful resource'})
mock_log.error.assert_called_with(CONTAINS('failure', 'error message'))
def test_resource_for_path_region_properties(self):
"""
If a project has a repository_url in pontoon.base.MOZILLA_REPOS,
resources_for_path should ignore files named
"region.properties".
"""
url = 'https://moz.example.com'
self.project.repositories.all().delete()
self.project.repositories.add(RepositoryFactory.create(url=url))
with patch('pontoon.sync.vcs.models.os', wraps=os) as mock_os, \
patch('pontoon.sync.vcs.models.MOZILLA_REPOS', [url]):
mock_os.walk.return_value = [
('/root', [], ['foo.pot', 'region.properties'])
]
assert_equal(
list(self.vcs_project.resources_for_path('/root')),
[os.path.join('/root', 'foo.pot')]
)
def test_filter_hidden_directories(self):
"""
We should filter out resources that are contained in the hidden paths.
"""
hidden_paths = (
('/root/.hidden_folder/templates', [], ('bar.pot',)),
('/root/templates', [], ('foo.pot',)),
)
with patch('pontoon.sync.vcs.models.os.walk', wraps=os, return_value=hidden_paths):
assert_equal(
list(self.vcs_project.resources_for_path('/root')),
['/root/templates/foo.pot']
)
def setUp(self):
self.now = aware_datetime(1970, 1, 1)
timezone_patch = patch("pontoon.sync.tasks.timezone")
self.mock_timezone = timezone_patch.start()
self.addCleanup(timezone_patch.stop)
self.mock_timezone.now.return_value = self.now
self.translated_locale = LocaleFactory.create(code="translated-locale")
self.inactive_locale = LocaleFactory.create(code="inactive-locale")
self.repository = RepositoryFactory()
self.db_project = ProjectFactory.create(
name="db-project",
locales=[self.translated_locale],
repositories=[self.repository],
)
self.main_db_resource = ResourceFactory.create(project=self.db_project,
path="main.lang",
format="lang")
self.other_db_resource = ResourceFactory.create(
project=self.db_project, path="other.lang", format="lang")
self.missing_db_resource = ResourceFactory.create(
project=self.db_project, path="missing.lang", format="lang")
self.main_db_entity = EntityFactory.create(
resource=self.main_db_resource,
string="Source String",
key="Source String",
obsolete=False,
)
self.other_db_entity = EntityFactory.create(
resource=self.other_db_resource,
string="Other Source String",
key="Other Source String",
obsolete=False,
)
self.main_db_translation = TranslationFactory.create(
entity=self.main_db_entity,
plural_form=None,
locale=self.translated_locale,
string="Translated String",
date=aware_datetime(1970, 1, 1),
approved=True,
extra={"tags": []},
)
# Load paths from the fake locale directory.
checkout_path_patch = patch.object(
Project,
"checkout_path",
new_callable=PropertyMock,
return_value=FAKE_CHECKOUT_PATH,
)
checkout_path_patch.start()
self.addCleanup(checkout_path_patch.stop)
vcs_changed_files = {
self.main_db_resource.path: [self.translated_locale],
self.other_db_resource.path: [self.translated_locale],
self.missing_db_resource.path: [self.translated_locale],
}
changed_files_patch = patch.object(
VCSProject,
"changed_files",
new_callable=PropertyMock,
return_value=vcs_changed_files,
)
changed_files_patch.start()
self.addCleanup(changed_files_patch.stop)
source_repository = patch.object(
Project,
"source_repository",
new_callable=PropertyMock,
return_value=self.db_project.repositories.all()[0],
)
source_repository.start()
self.addCleanup(source_repository.stop)
self.vcs_project = VCSProject(self.db_project)
self.main_vcs_resource = self.vcs_project.resources[
self.main_db_resource.path]
self.other_vcs_resource = self.vcs_project.resources[
self.other_db_resource.path]
self.missing_vcs_resource = self.vcs_project.resources[
self.missing_db_resource.path]
self.main_vcs_entity = self.main_vcs_resource.entities["Source String"]
self.main_vcs_translation = self.main_vcs_entity.translations[
"translated-locale"]
# Mock VCSResource.save() for each resource to avoid altering
# the filesystem.
resource_save_patch = patch.object(VCSResource, "save")
resource_save_patch.start()
self.addCleanup(resource_save_patch.stop)
self.changeset = ChangeSet(
self.db_project,
self.vcs_project,
aware_datetime(1970, 1, 1),
self.translated_locale,
)
class VCSProjectTests(TestCase):
def setUp(self):
# Force the checkout path to point to a test directory to make
# resource file loading pass during tests.
checkout_path_patch = patch.object(Project,
'checkout_path',
new_callable=PropertyMock,
return_value=os.path.join(
TEST_CHECKOUT_PATH,
'no_resources_test'))
self.mock_checkout_path = checkout_path_patch.start()
self.addCleanup(checkout_path_patch.stop)
self.project = ProjectFactory.create()
self.vcs_project = VCSProject(self.project)
def test_relative_resource_paths(self):
with patch.object(VCSProject,
'source_directory_path',
new_callable=PropertyMock,
return_value='/root/'):
self.vcs_project.resources_for_path = Mock(
return_value=['/root/foo.po', '/root/meh/bar.po'])
assert_equal(list(self.vcs_project.relative_resource_paths()),
['foo.po', 'meh/bar.po'])
def test_relative_resource_paths_pot(self):
"""
If a resource ends in .pot, replace the extension with .po since
relative paths are used within non-source locales that do not
have .pot files.
"""
with patch.object(VCSProject,
'source_directory_path',
new_callable=PropertyMock,
return_value='/root/'):
self.vcs_project.resources_for_path = Mock(
return_value=['/root/foo.pot', '/root/meh/bar.pot'])
assert_equal(list(self.vcs_project.relative_resource_paths()),
['foo.po', 'meh/bar.po'])
def test_source_directory_path_no_resource(self):
"""
When searching for source directories, do not match directories that
do not contain resource files.
"""
checkout_path = os.path.join(TEST_CHECKOUT_PATH, 'no_resources_test')
self.mock_checkout_path.return_value = checkout_path
assert_equal(
self.vcs_project.source_directory_path,
os.path.join(checkout_path, 'real_resources', 'templates'))
def test_source_directory_scoring_templates(self):
"""
When searching for source directories, prefer directories named
`templates` over all others.
"""
checkout_path = os.path.join(TEST_CHECKOUT_PATH,
'scoring_templates_test')
self.mock_checkout_path.return_value = checkout_path
assert_equal(self.vcs_project.source_directory_path,
os.path.join(checkout_path, 'templates'))
def test_source_directory_scoring_en_US(self):
"""
When searching for source directories, prefer directories named
`en-US` over others besides `templates`.
"""
checkout_path = os.path.join(TEST_CHECKOUT_PATH, 'scoring_en_US_test')
self.mock_checkout_path.return_value = checkout_path
assert_equal(self.vcs_project.source_directory_path,
os.path.join(checkout_path, 'en-US'))
def test_source_directory_scoring_source_files(self):
"""
When searching for source directories, prefer directories with
source-only formats over all others.
"""
checkout_path = os.path.join(TEST_CHECKOUT_PATH,
'scoring_source_files_test')
self.mock_checkout_path.return_value = checkout_path
assert_equal(
self.vcs_project.source_directory_path,
os.path.join(checkout_path, 'en') # en has pot files in it
)
def test_resources_parse_error(self):
"""
If VCSResource() raises a ParseError while loading, log an error
and skip the resource.
"""
self.vcs_project.relative_resource_paths = Mock(
return_value=['failure', 'success'])
# Fail only if the path is failure so we can test the ignore.
def vcs_resource_constructor(project, path, locales=None):
if path == 'failure':
raise ParseError('error message')
else:
return 'successful resource'
changed_vcs_resources = {'success': [], 'failure': []}
with patch('pontoon.sync.vcs.models.VCSResource') as MockVCSResource, \
patch('pontoon.sync.vcs.models.log') as mock_log, \
patch.object(
VCSProject, 'changed_files', new_callable=PropertyMock,
return_value=changed_vcs_resources
):
MockVCSResource.side_effect = vcs_resource_constructor
assert_equal(self.vcs_project.resources,
{'success': 'successful resource'})
mock_log.error.assert_called_with(
CONTAINS('failure', 'error message'))
def test_resource_for_path_region_properties(self):
"""
If a project has a repository_url in pontoon.base.MOZILLA_REPOS,
resources_for_path should ignore files named
"region.properties".
"""
url = 'https://moz.example.com'
self.project.repositories.all().delete()
self.project.repositories.add(RepositoryFactory.create(url=url))
with patch('pontoon.sync.vcs.models.scandir', wraps=scandir) as mock_scandir, \
patch(
'pontoon.sync.vcs.models.MOZILLA_REPOS', [url]
):
mock_scandir.walk.return_value = [
('/root', [], ['foo.pot', 'region.properties'])
]
assert_equal(list(self.vcs_project.resources_for_path('/root')),
[os.path.join('/root', 'foo.pot')])
def test_filter_hidden_directories(self):
"""
We should filter out resources that are contained in the hidden paths.
"""
hidden_paths = (
('/root/.hidden_folder/templates', [], ('bar.pot', )),
('/root/templates', [], ('foo.pot', )),
)
with patch('pontoon.sync.vcs.models.scandir.walk',
wraps=scandir,
return_value=hidden_paths):
assert_equal(list(self.vcs_project.resources_for_path('/root')),
['/root/templates/foo.pot'])
def sync_translations(
self,
project_pk,
project_sync_log_pk,
now,
added_paths=None,
removed_paths=None,
changed_paths=None,
new_entities=None,
locale=None,
no_pull=False,
no_commit=False,
full_scan=False,
):
db_project = get_or_fail(
Project,
pk=project_pk,
message="Could not sync project with pk={0}, not found.".format(project_pk),
)
repos = db_project.translation_repositories()
repo_pk = repos[0].pk
repo = get_or_fail(
Repository,
pk=repo_pk,
message="Could not sync repo with pk={0}, not found.".format(repo_pk),
)
project_sync_log = get_or_fail(
ProjectSyncLog,
pk=project_sync_log_pk,
message=(
"Could not sync project {0}, log with pk={1} not found.".format(
db_project.slug, project_sync_log_pk
)
),
)
log.info("Syncing translations for project: {}".format(db_project.slug))
repo_sync_log = RepositorySyncLog.objects.create(
project_sync_log=project_sync_log, repository=repo, start_time=timezone.now()
)
if locale:
locales = db_project.locales.filter(pk=locale.pk)
else:
locales = db_project.locales.all()
if not locales:
log.info(
"Skipping syncing translations for project {0}, no locales to sync "
"found within.".format(db_project.slug)
)
repo_sync_log.end()
return
# If project repositories have API access, we can retrieve latest commit hashes and detect
# changed locales before the expensive VCS pull/clone operations. When performing full scan,
# we still need to sync all locales.
if not full_scan:
locales = get_changed_locales(db_project, locales, now)
readonly_locales = db_project.locales.filter(project_locale__readonly=True)
added_and_changed_resources = db_project.resources.filter(
path__in=list(added_paths or []) + list(changed_paths or [])
).distinct()
# We should also sync files for which source file change - but only for read-only locales.
# See bug 1372151 for more details.
if added_and_changed_resources:
changed_locales_pks = [l.pk for l in locales]
readonly_locales_pks = [l.pk for l in readonly_locales]
locales = db_project.locales.filter(
pk__in=changed_locales_pks + readonly_locales_pks
)
# Pull VCS changes in case we're on a different worker than the one
# sync started on.
if not no_pull:
log.info("Pulling changes for project {0} started.".format(db_project.slug))
repos_changed, repo_locales = pull_changes(db_project, locales)
repos = repos.filter(pk__in=repo_locales.keys())
log.info("Pulling changes for project {0} complete.".format(db_project.slug))
# If none of the repos has changed since the last sync and there are
# no Pontoon-side changes for this project, quit early.
if (
not full_scan
and not db_project.needs_sync
and not repos_changed
and not (added_paths or removed_paths or changed_paths)
):
log.info("Skipping project {0}, no changes detected.".format(db_project.slug))
repo_sync_log.end()
return
vcs_project = VCSProject(
db_project,
now,
locales=locales,
repo_locales=repo_locales,
added_paths=added_paths,
changed_paths=changed_paths,
full_scan=full_scan,
)
synced_locales = set()
failed_locales = set()
# Store newly added locales and locales with newly added resources
new_locales = []
for locale in locales:
try:
with transaction.atomic():
# Sets VCSProject.synced_locales, needed to skip early
if not vcs_project.synced_locales:
vcs_project.resources
# Skip all locales if none of the them has anything to sync
if len(vcs_project.synced_locales) == 0:
break
# Skip locales that have nothing to sync
if (
vcs_project.synced_locales
and locale not in vcs_project.synced_locales
):
continue
changeset = ChangeSet(db_project, vcs_project, now, locale)
update_translations(db_project, vcs_project, locale, changeset)
changeset.execute()
created = update_translated_resources(db_project, vcs_project, locale)
if created:
new_locales.append(locale.pk)
update_locale_project_locale_stats(locale, db_project)
# Clear out the "has_changed" markers now that we've finished
# syncing.
(
ChangedEntityLocale.objects.filter(
entity__resource__project=db_project,
locale=locale,
when__lte=now,
).delete()
)
# Perform the commit last so that, if it succeeds, there is
# nothing after it to fail.
if (
not no_commit
and locale in changeset.locales_to_commit
and locale not in readonly_locales
):
commit_changes(db_project, vcs_project, changeset, locale)
log.info(
"Synced locale {locale} for project {project}.".format(
locale=locale.code, project=db_project.slug,
)
)
synced_locales.add(locale.code)
except CommitToRepositoryException as err:
# Transaction aborted, log and move on to the next locale.
log.warning(
"Failed to sync locale {locale} for project {project} due to "
"commit error: {error}".format(
locale=locale.code, project=db_project.slug, error=err,
)
)
failed_locales.add(locale.code)
# If sources have changed, update stats for all locales.
if added_paths or removed_paths or changed_paths:
for locale in db_project.locales.all():
# Already synced.
if locale.code in synced_locales:
continue
# We have files: update all translated resources.
if locale in locales:
created = update_translated_resources(db_project, vcs_project, locale)
if created:
new_locales.append[locale.pk]
# We don't have files: we can still update asymmetric translated resources.
else:
update_translated_resources_no_files(
db_project, locale, added_and_changed_resources,
)
update_locale_project_locale_stats(locale, db_project)
synced_locales.add(locale.code)
log.info(
"Synced source changes for locale {locale} for project {project}.".format(
locale=locale.code, project=db_project.slug,
)
)
db_project.aggregate_stats()
if synced_locales:
log.info(
"Synced translations for project {0} in locales {1}.".format(
db_project.slug, ",".join(synced_locales)
)
)
elif failed_locales:
log.info(
"Failed to sync translations for project {0} due to commit error.".format(
db_project.slug
)
)
else:
log.info(
"Skipping syncing translations for project {0}, none of the locales "
"has anything to sync.".format(db_project.slug)
)
for r in repos:
r.set_last_synced_revisions(
locales=repo_locales[r.pk].exclude(code__in=failed_locales)
)
repo_sync_log.end()
if db_project.pretranslation_enabled:
# Pretranslate all entities for newly added locales
# and locales with newly added resources
if len(new_locales):
pretranslate(db_project, locales=new_locales)
locales = db_project.locales.exclude(pk__in=new_locales).values_list(
"pk", flat=True
)
# Pretranslate newly added entities for all locales
if new_entities and locales:
new_entities = list(set(new_entities))
pretranslate(db_project, locales=locales, entities=new_entities)
def sync_project_repo(self,
project_pk,
repo_pk,
project_sync_log_pk,
now,
obsolete_vcs_entities=None,
obsolete_vcs_resources=None,
new_paths=None,
locale=None,
no_pull=False,
no_commit=False,
full_scan=False):
db_project = get_or_fail(
Project,
pk=project_pk,
message='Could not sync project with pk={0}, not found.'.format(
project_pk))
repo = get_or_fail(
Repository,
pk=repo_pk,
message='Could not sync repo with pk={0}, not found.'.format(repo_pk))
project_sync_log = get_or_fail(
ProjectSyncLog,
pk=project_sync_log_pk,
message=(
'Could not sync project {0}, log with pk={1} not found.'.format(
db_project.slug, project_sync_log_pk)))
log.info('Syncing repo: {}'.format(repo.url))
repo_sync_log = RepositorySyncLog.objects.create(
project_sync_log=project_sync_log,
repository=repo,
start_time=timezone.now())
# Pull VCS changes in case we're on a different worker than the one
# sync started on.
if not no_pull:
pull_changes(db_project)
if locale:
locales = [locale]
else:
locales = repo.locales
# Cannot skip earlier - repo.locales is only available after pull_changes()
if not locales:
log.debug(
'Skipping repo `{0}` for project {1}, no locales to sync found within.'
.format(repo.url, db_project.slug))
repo_sync_log.end()
return
vcs_project = VCSProject(
db_project,
locales=locales,
obsolete_entities_paths=Resource.objects.obsolete_entities_paths(
obsolete_vcs_entities),
new_paths=new_paths,
full_scan=full_scan)
for locale in locales:
try:
with transaction.atomic():
# Skip locales that have nothing to sync
if vcs_project.synced_locales and locale not in vcs_project.synced_locales:
if obsolete_vcs_entities or obsolete_vcs_resources:
update_locale_project_locale_stats(locale, db_project)
continue
changeset = ChangeSet(db_project, vcs_project, now,
obsolete_vcs_entities,
obsolete_vcs_resources)
update_translations(db_project, vcs_project, locale, changeset)
changeset.execute()
update_translated_resources(db_project, vcs_project, changeset,
locale)
# Skip if none of the locales has anything to sync
# VCSProject.synced_locales is set on a first call to
# VCSProject.resources, which is set in
# pontoon.sync.core.update_translated_resources()
if len(vcs_project.synced_locales) == 0:
if obsolete_vcs_entities or obsolete_vcs_resources:
for l in locales:
update_locale_project_locale_stats(l, db_project)
db_project.aggregate_stats()
log.info(
'Skipping repo `{0}` for project {1}, none of the locales has anything to sync.'
.format(repo.url, db_project.slug))
repo_sync_log.end()
return
update_locale_project_locale_stats(locale, db_project)
# Clear out the "has_changed" markers now that we've finished
# syncing.
(ChangedEntityLocale.objects.filter(
entity__resource__project=db_project,
locale=locale,
when__lte=now).delete())
# Clean up any duplicate approvals at the end of sync right
# before we commit the transaction to avoid race conditions.
with connection.cursor() as cursor:
cursor.execute(
"""
UPDATE base_translation AS b
SET approved = FALSE, approved_date = NULL
WHERE
id IN
(SELECT trans.id FROM base_translation AS trans
LEFT JOIN base_entity AS ent ON ent.id = trans.entity_id
LEFT JOIN base_resource AS res ON res.id = ent.resource_id
WHERE locale_id = %(locale_id)s
AND res.project_id = %(project_id)s)
AND approved_date !=
(SELECT max(approved_date)
FROM base_translation
WHERE entity_id = b.entity_id
AND locale_id = b.locale_id
AND (plural_form = b.plural_form OR plural_form IS NULL));
""", {
'locale_id': locale.id,
'project_id': db_project.id
})
# Perform the commit last so that, if it succeeds, there is
# nothing after it to fail.
if not no_commit and locale in changeset.locales_to_commit:
commit_changes(db_project, vcs_project, changeset, locale)
except CommitToRepositoryException as err:
# Transaction aborted, log and move on to the next locale.
log.warning(
'Failed to sync locale {locale} for project {project} due to '
'commit error: {error}'.format(
locale=locale.code,
project=db_project.slug,
error=err,
))
with transaction.atomic():
db_project.aggregate_stats()
log.info('Synced translations for project {0} in locales {1}.'.format(
db_project.slug,
','.join(locale.code for locale in vcs_project.synced_locales)))
repo_sync_log.end()
def sync_translations(self,
project_pk,
project_sync_log_pk,
now,
project_changes=None,
obsolete_vcs_resources=None,
new_paths=None,
locale=None,
no_pull=False,
no_commit=False,
full_scan=False):
db_project = get_or_fail(
Project,
pk=project_pk,
message='Could not sync project with pk={0}, not found.'.format(
project_pk))
repos = db_project.translation_repositories()
repo_pk = repos[0].pk
repo = get_or_fail(
Repository,
pk=repo_pk,
message='Could not sync repo with pk={0}, not found.'.format(repo_pk))
project_sync_log = get_or_fail(
ProjectSyncLog,
pk=project_sync_log_pk,
message=(
'Could not sync project {0}, log with pk={1} not found.'.format(
db_project.slug, project_sync_log_pk)))
log.info('Syncing translations for project: {}'.format(db_project.slug))
repo_sync_log = RepositorySyncLog.objects.create(
project_sync_log=project_sync_log,
repository=repo,
start_time=timezone.now())
if locale:
locales = db_project.locales.filter(pk=locale.pk)
else:
locales = db_project.locales.all()
if not locales:
log.info(
'Skipping syncing translations for project {0}, no locales to sync found within.'
.format(db_project.slug))
repo_sync_log.end()
return
# If project repositories have API access, we can retrieve latest commit hashes and detect
# changed locales before the expensive VCS pull/clone operations. When performing full scan,
# we still need to sync all locales.
if not full_scan:
locales = get_changed_locales(db_project, locales, now)
# Pull VCS changes in case we're on a different worker than the one
# sync started on.
if not no_pull:
log.info('Pulling changes for project {0} started.'.format(
db_project.slug))
repos_changed, repo_locales = pull_changes(db_project, locales)
repos = repos.filter(pk__in=repo_locales.keys())
log.info('Pulling changes for project {0} complete.'.format(
db_project.slug))
changed_resources = []
obsolete_vcs_entities = []
if project_changes:
updated_entity_pks = []
for locale_code, db_entity, vcs_entity in project_changes['update_db']:
updated_entity_pks.append(db_entity.pk)
obsolete_entity_pks = project_changes['obsolete_db']
changed_resources = db_project.resources.filter(
Q(entities__date_created=now)
| Q(entities__pk__in=updated_entity_pks +
obsolete_entity_pks)).distinct()
obsolete_vcs_entities = project_changes['obsolete_db']
# If none of the repos has changed since the last sync and there are
# no Pontoon-side changes for this project, quit early.
if (not full_scan and not db_project.needs_sync and not repos_changed
and not (changed_resources or obsolete_vcs_resources)):
log.info('Skipping project {0}, no changes detected.'.format(
db_project.slug))
repo_sync_log.end()
return
obsolete_entities_paths = (
Resource.objects.obsolete_entities_paths(obsolete_vcs_entities)
if obsolete_vcs_entities else None)
vcs_project = VCSProject(db_project,
now,
locales=locales,
repo_locales=repo_locales,
obsolete_entities_paths=obsolete_entities_paths,
new_paths=new_paths,
full_scan=full_scan)
synced_locales = set()
failed_locales = set()
for locale in locales:
try:
with transaction.atomic():
# Sets VCSProject.synced_locales, needed to skip early
if not vcs_project.synced_locales:
vcs_project.resources
# Skip all locales if none of the them has anything to sync
if len(vcs_project.synced_locales) == 0:
break
# Skip locales that have nothing to sync
if vcs_project.synced_locales and locale not in vcs_project.synced_locales:
continue
changeset = ChangeSet(db_project, vcs_project, now, locale)
update_translations(db_project, vcs_project, locale, changeset)
changeset.execute()
update_translated_resources(db_project, vcs_project, locale)
update_locale_project_locale_stats(locale, db_project)
# Clear out the "has_changed" markers now that we've finished
# syncing.
(ChangedEntityLocale.objects.filter(
entity__resource__project=db_project,
locale=locale,
when__lte=now).delete())
# Clean up any duplicate approvals at the end of sync right
# before we commit the transaction to avoid race conditions.
with connection.cursor() as cursor:
cursor.execute(
"""
UPDATE base_translation AS b
SET approved = FALSE, approved_date = NULL
WHERE
id IN
(SELECT trans.id FROM base_translation AS trans
LEFT JOIN base_entity AS ent ON ent.id = trans.entity_id
LEFT JOIN base_resource AS res ON res.id = ent.resource_id
WHERE locale_id = %(locale_id)s
AND res.project_id = %(project_id)s)
AND approved_date !=
(SELECT max(approved_date)
FROM base_translation
WHERE entity_id = b.entity_id
AND locale_id = b.locale_id
AND (plural_form = b.plural_form OR plural_form IS NULL));
""", {
'locale_id': locale.id,
'project_id': db_project.id
})
# Perform the commit last so that, if it succeeds, there is
# nothing after it to fail.
if not no_commit and locale in changeset.locales_to_commit:
commit_changes(db_project, vcs_project, changeset, locale)
log.info(
'Synced locale {locale} for project {project}.'.format(
locale=locale.code,
project=db_project.slug,
))
synced_locales.add(locale.code)
except CommitToRepositoryException as err:
# Transaction aborted, log and move on to the next locale.
log.warning(
'Failed to sync locale {locale} for project {project} due to '
'commit error: {error}'.format(
locale=locale.code,
project=db_project.slug,
error=err,
))
failed_locales.add(locale.code)
# If sources have changed, update stats for all locales.
if changed_resources or obsolete_vcs_resources:
for locale in db_project.locales.all():
# Already synced.
if locale.code in synced_locales:
continue
# We have files: update all translated resources.
if locale in locales:
update_translated_resources(db_project, vcs_project, locale)
# We don't have files: we can still update asymmetric translated resources.
else:
update_translated_resources_no_files(db_project, locale,
changed_resources)
update_locale_project_locale_stats(locale, db_project)
synced_locales.add(locale.code)
log.info(
'Synced source changes for locale {locale} for project {project}.'
.format(
locale=locale.code,
project=db_project.slug,
))
db_project.aggregate_stats()
if synced_locales:
log.info('Synced translations for project {0} in locales {1}.'.format(
db_project.slug, ','.join(synced_locales)))
elif failed_locales:
log.info(
'Failed to sync translations for project {0} due to commit error.'.
format(db_project.slug))
else:
log.info(
'Skipping syncing translations for project {0}, none of the locales '
'has anything to sync.'.format(db_project.slug))
for r in repos:
r.set_last_synced_revisions(locales=repo_locales[r.pk].exclude(
code__in=failed_locales))
repo_sync_log.end()
def sync_project_repo(self,
project_pk,
repo_pk,
project_sync_log_pk,
now,
obsolete_vcs=None,
no_pull=False,
no_commit=False):
db_project = get_or_fail(
Project,
pk=project_pk,
message='Could not sync project with pk={0}, not found.'.format(
project_pk))
repo = get_or_fail(
Repository,
pk=repo_pk,
message='Could not sync repo with pk={0}, not found.'.format(
project_pk))
project_sync_log = get_or_fail(
ProjectSyncLog,
pk=project_sync_log_pk,
message=(
'Could not sync project {0}, log with pk={1} not found.'.format(
db_project.slug, project_sync_log_pk)))
repo_sync_log = RepositorySyncLog.objects.create(
project_sync_log=project_sync_log,
repository=repo,
start_time=timezone.now())
# Pull VCS changes in case we're on a different worker than the one
# sync started on.
if not no_pull:
pull_changes(db_project)
if len(repo.locales) < 1:
log.warning(
'Could not sync repo `{0}`, no locales found within.'.format(
repo.url))
repo_sync_log.end_time = timezone.now()
repo_sync_log.save(update_fields=['end_time'])
return
vcs_project = VCSProject(db_project, locales=repo.locales)
for locale in repo.locales:
try:
with transaction.atomic():
changeset = ChangeSet(db_project, vcs_project, now,
obsolete_vcs)
update_translations(db_project, vcs_project, locale, changeset)
changeset.execute()
update_translated_resources(db_project, vcs_project, changeset,
locale)
# Clear out the "has_changed" markers now that we've finished
# syncing.
(ChangedEntityLocale.objects.filter(
entity__resource__project=db_project,
locale=locale,
when__lte=now).delete())
db_project.has_changed = False
db_project.save(update_fields=['has_changed'])
# Clean up any duplicate approvals at the end of sync right
# before we commit the transaction to avoid race conditions.
with connection.cursor() as cursor:
cursor.execute(
"""
UPDATE base_translation AS b
SET approved = FALSE, approved_date = NULL
WHERE
id IN
(SELECT trans.id FROM base_translation AS trans
LEFT JOIN base_entity AS ent ON ent.id = trans.entity_id
LEFT JOIN base_resource AS res ON res.id = ent.resource_id
WHERE locale_id = %(locale_id)s
AND res.project_id = %(project_id)s)
AND approved_date !=
(SELECT max(approved_date)
FROM base_translation
WHERE entity_id = b.entity_id
AND locale_id = b.locale_id
AND (plural_form = b.plural_form OR plural_form IS NULL));
""", {
'locale_id': locale.id,
'project_id': db_project.id
})
# Perform the commit last so that, if it succeeds, there is
# nothing after it to fail.
if not no_commit and locale in changeset.locales_to_commit:
commit_changes(db_project, vcs_project, changeset, locale)
except CommitToRepositoryException as err:
# Transaction aborted, log and move on to the next locale.
log.warning(
'Failed to sync locale {locale} for project {project} due to '
'commit error: {error}'.format(
locale=locale.code,
project=db_project.slug,
error=err,
))
repo_sync_log.end_time = timezone.now()
repo_sync_log.save()
log.info('Synced translations for project {0} in locales {1}.'.format(
db_project.slug, ','.join(locale.code for locale in repo.locales)))
def get_download_content(slug, code, part):
"""
Get content of the file to be downloaded.
:arg str slug: Project slug.
:arg str code: Locale code.
:arg str part: Resource path or Subpage name.
"""
# Avoid circular import; someday we should refactor to avoid.
from pontoon.sync import formats
from pontoon.sync.utils import source_to_locale_path
from pontoon.sync.vcs.models import VCSProject
from pontoon.base.models import Entity, Locale, Project, Resource
project = get_object_or_404(Project, slug=slug)
locale = get_object_or_404(Locale, code=code)
vcs_project = VCSProject(project, locales=[locale])
# Download a ZIP of all files if project has > 1 and < 10 resources
resources = Resource.objects.filter(project=project,
translatedresources__locale=locale)
isZipable = 1 < len(resources) < 10
if isZipable:
s = io.BytesIO()
zf = zipfile.ZipFile(s, "w")
# Download a single file if project has 1 or >= 10 resources
else:
relative_path = _get_relative_path_from_part(slug, part)
resources = [
get_object_or_404(Resource, project__slug=slug, path=relative_path)
]
locale_prefixes = project.repositories
if not project.configuration_file:
locale_prefixes = locale_prefixes.filter(
permalink_prefix__contains="{locale_code}")
locale_prefixes = locale_prefixes.values_list("permalink_prefix",
flat=True).distinct()
source_prefixes = project.repositories.values_list("permalink_prefix",
flat=True).distinct()
for resource in resources:
# Get locale file
dirnames = set([locale.code, locale.code.replace("-", "_")])
locale_path = _download_file(locale_prefixes, dirnames, vcs_project,
resource.path)
if not locale_path and not resource.is_asymmetric:
return None, None
# Get source file if needed
source_path = None
if resource.is_asymmetric:
dirnames = VCSProject.SOURCE_DIR_NAMES
source_path = _download_file(source_prefixes, dirnames,
vcs_project, resource.path)
if not source_path:
return None, None
# If locale file doesn't exist, create it
if not locale_path:
extension = os.path.splitext(resource.path)[1]
with tempfile.NamedTemporaryFile(
prefix="strings" if extension == ".xml" else "",
suffix=extension,
delete=False,
) as temp:
temp.flush()
locale_path = temp.name
# Update file from database
resource_file = formats.parse(locale_path, source_path)
entities_dict = {}
entities_qs = Entity.objects.filter(
changedentitylocale__locale=locale,
resource__project=project,
resource__path=resource.path,
obsolete=False,
)
for e in entities_qs:
entities_dict[e.key] = e.translation_set.filter(approved=True,
locale=locale)
for vcs_translation in resource_file.translations:
key = vcs_translation.key
if key in entities_dict:
entity = entities_dict[key]
vcs_translation.update_from_db(entity)
resource_file.save(locale)
if not locale_path:
return None, None
if isZipable:
zf.write(locale_path, source_to_locale_path(resource.path))
else:
with codecs.open(locale_path, "r", "utf-8") as f:
content = f.read()
filename = os.path.basename(source_to_locale_path(resource.path))
# Remove temporary files
os.remove(locale_path)
if source_path:
os.remove(source_path)
if isZipable:
zf.close()
content = s.getvalue()
filename = project.slug + ".zip"
return content, filename
def setUp(self):
self.now = aware_datetime(1970, 1, 1)
timezone_patch = patch('pontoon.sync.tasks.timezone')
self.mock_timezone = timezone_patch.start()
self.addCleanup(timezone_patch.stop)
self.mock_timezone.now.return_value = self.now
self.translated_locale = LocaleFactory.create(code='translated-locale')
self.inactive_locale = LocaleFactory.create(code='inactive-locale')
self.repository = RepositoryFactory()
self.db_project = ProjectFactory.create(
name='db-project',
locales=[self.translated_locale],
repositories=[self.repository])
self.main_db_resource = ResourceFactory.create(project=self.db_project,
path='main.lang',
format='lang')
self.other_db_resource = ResourceFactory.create(
project=self.db_project, path='other.lang', format='lang')
self.missing_db_resource = ResourceFactory.create(
project=self.db_project, path='missing.lang', format='lang')
self.main_db_entity = EntityFactory.create(
resource=self.main_db_resource,
string='Source String',
key='Source String',
obsolete=False)
self.other_db_entity = EntityFactory.create(
resource=self.other_db_resource,
string='Other Source String',
key='Other Source String',
obsolete=False)
self.main_db_translation = TranslationFactory.create(
entity=self.main_db_entity,
plural_form=None,
locale=self.translated_locale,
string='Translated String',
date=aware_datetime(1970, 1, 1),
approved=True,
extra={'tags': []})
# Load paths from the fake locale directory.
checkout_path_patch = patch.object(Project,
'checkout_path',
new_callable=PropertyMock,
return_value=FAKE_CHECKOUT_PATH)
checkout_path_patch.start()
self.addCleanup(checkout_path_patch.stop)
self.vcs_project = VCSProject(self.db_project)
self.main_vcs_resource = self.vcs_project.resources[
self.main_db_resource.path]
self.other_vcs_resource = self.vcs_project.resources[
self.other_db_resource.path]
self.missing_vcs_resource = self.vcs_project.resources[
self.missing_db_resource.path]
self.main_vcs_entity = self.main_vcs_resource.entities['Source String']
self.main_vcs_translation = self.main_vcs_entity.translations[
'translated-locale']
# Mock VCSResource.save() for each resource to avoid altering
# the filesystem.
resource_save_patch = patch.object(VCSResource, 'save')
resource_save_patch.start()
self.addCleanup(resource_save_patch.stop)
self.changeset = ChangeSet(self.db_project, self.vcs_project,
aware_datetime(1970, 1, 1))
def handle_upload_content(slug, code, part, f, user):
"""
Update translations in the database from uploaded file.
:arg str slug: Project slug.
:arg str code: Locale code.
:arg str part: Resource path or Subpage name.
:arg UploadedFile f: UploadedFile instance.
:arg User user: User uploading the file.
"""
# Avoid circular import; someday we should refactor to avoid.
from pontoon.sync import formats
from pontoon.sync.changeset import ChangeSet
from pontoon.sync.vcs.models import VCSProject
from pontoon.base.models import (
ChangedEntityLocale,
Entity,
Locale,
Project,
Resource,
TranslatedResource,
Translation,
)
relative_path = _get_relative_path_from_part(slug, part)
project = get_object_or_404(Project, slug=slug)
locale = get_object_or_404(Locale, code=code)
resource = get_object_or_404(Resource,
project__slug=slug,
path=relative_path)
# Store uploaded file to a temporary file and parse it
extension = os.path.splitext(f.name)[1]
with tempfile.NamedTemporaryFile(suffix=extension) as temp:
for chunk in f.chunks():
temp.write(chunk)
temp.flush()
resource_file = formats.parse(temp.name)
# Update database objects from file
changeset = ChangeSet(project, VCSProject(project, locales=[locale]),
timezone.now())
entities_qs = Entity.objects.filter(
resource__project=project,
resource__path=relative_path,
obsolete=False).prefetch_related(
Prefetch('translation_set',
queryset=Translation.objects.filter(locale=locale),
to_attr='db_translations')).prefetch_related(
Prefetch(
'translation_set',
queryset=Translation.objects.filter(
locale=locale,
approved_date__lte=timezone.now()),
to_attr='db_translations_approved_before_sync'))
entities_dict = {entity.key: entity for entity in entities_qs}
for vcs_translation in resource_file.translations:
key = vcs_translation.key
if key in entities_dict:
entity = entities_dict[key]
changeset.update_entity_translations_from_vcs(
entity, locale.code, vcs_translation, user,
entity.db_translations,
entity.db_translations_approved_before_sync)
changeset.bulk_create_translations()
changeset.bulk_update_translations()
changeset.bulk_create_translaton_memory_entries()
TranslatedResource.objects.get(resource=resource,
locale=locale).calculate_stats()
# Mark translations as changed
changed_entities = {}
existing = ChangedEntityLocale.objects.values_list('entity',
'locale').distinct()
for t in changeset.changed_translations:
key = (t.entity.pk, t.locale.pk)
# Remove duplicate changes to prevent unique constraint violation
if key not in existing:
changed_entities[key] = ChangedEntityLocale(entity=t.entity,
locale=t.locale)
ChangedEntityLocale.objects.bulk_create(changed_entities.values())
# Update latest translation
if changeset.translations_to_create:
changeset.translations_to_create[-1].update_latest_translation()
def handle_upload_content(slug, code, part, f, user):
"""
Update translations in the database from uploaded file.
:arg str slug: Project slug.
:arg str code: Locale code.
:arg str part: Resource path or Subpage name.
:arg UploadedFile f: UploadedFile instance.
:arg User user: User uploading the file.
"""
# Avoid circular import; someday we should refactor to avoid.
from pontoon.sync import formats
from pontoon.sync.changeset import ChangeSet
from pontoon.sync.vcs.models import VCSProject
from pontoon.base.models import (
ChangedEntityLocale,
Entity,
Locale,
Project,
Resource,
TranslatedResource,
Translation,
)
relative_path = _get_relative_path_from_part(slug, part)
project = get_object_or_404(Project, slug=slug)
locale = get_object_or_404(Locale, code=code)
resource = get_object_or_404(Resource,
project__slug=slug,
path=relative_path)
# Store uploaded file to a temporary file and parse it
extension = os.path.splitext(f.name)[1]
with tempfile.NamedTemporaryFile(
prefix="strings" if extension == ".xml" else "",
suffix=extension,
) as temp:
for chunk in f.chunks():
temp.write(chunk)
temp.flush()
resource_file = formats.parse(temp.name)
# Update database objects from file
changeset = ChangeSet(project, VCSProject(project, locales=[locale]),
timezone.now())
entities_qs = (Entity.objects.filter(
resource__project=project,
resource__path=relative_path,
obsolete=False).prefetch_related(
Prefetch(
"translation_set",
queryset=Translation.objects.filter(locale=locale),
to_attr="db_translations",
)).prefetch_related(
Prefetch(
"translation_set",
queryset=Translation.objects.filter(
locale=locale, approved_date__lte=timezone.now()),
to_attr="db_translations_approved_before_sync",
)))
entities_dict = {entity.key: entity for entity in entities_qs}
for vcs_translation in resource_file.translations:
key = vcs_translation.key
if key in entities_dict:
entity = entities_dict[key]
changeset.update_entity_translations_from_vcs(
entity,
locale.code,
vcs_translation,
user,
entity.db_translations,
entity.db_translations_approved_before_sync,
)
changeset.bulk_create_translations()
changeset.bulk_update_translations()
changeset.bulk_log_actions()
if changeset.changed_translations:
# Update 'active' status of all changed translations and their siblings,
# i.e. translations of the same entity to the same locale.
changed_pks = {t.pk for t in changeset.changed_translations}
(Entity.objects.filter(
translation__pk__in=changed_pks).reset_active_translations(
locale=locale))
# Run checks and create TM entries for translations that pass them
valid_translations = changeset.bulk_check_translations()
changeset.bulk_create_translation_memory_entries(valid_translations)
# Remove any TM entries of translations that got rejected
changeset.bulk_remove_translation_memory_entries()
TranslatedResource.objects.get(resource=resource,
locale=locale).calculate_stats()
# Mark translations as changed
changed_entities = {}
existing = ChangedEntityLocale.objects.values_list("entity",
"locale").distinct()
for t in changeset.changed_translations:
key = (t.entity.pk, t.locale.pk)
# Remove duplicate changes to prevent unique constraint violation
if key not in existing:
changed_entities[key] = ChangedEntityLocale(entity=t.entity,
locale=t.locale)
ChangedEntityLocale.objects.bulk_create(changed_entities.values())
# Update latest translation
if changeset.translations_to_create:
changeset.translations_to_create[-1].update_latest_translation()
