def setUp(self):
# Force the checkout path to point to a test directory to make
# resource file loading pass during tests.
checkout_path_patch = patch.object(Project,
'checkout_path',
new_callable=PropertyMock,
return_value=os.path.join(
TEST_CHECKOUT_PATH,
'no_resources_test'))
self.mock_checkout_path = checkout_path_patch.start()
self.addCleanup(checkout_path_patch.stop)
self.project = ProjectFactory.create()
self.vcs_project = VCSProject(self.project)
def sync_project(db_project, now):
# Only load source resources for updating entities.
vcs_project = VCSProject(db_project, locales=[])
with transaction.atomic():
update_resources(db_project, vcs_project)
changeset = ChangeSet(db_project, vcs_project, now)
update_entities(db_project, vcs_project, changeset)
changeset.execute()
def setUp(self):
# Force the checkout path to point to a test directory to make
# resource file loading pass during tests.
checkout_path_patch = patch.object(
Project,
'checkout_path',
new_callable=PropertyMock,
return_value=os.path.join(TEST_CHECKOUT_PATH, 'no_resources_test')
)
self.mock_checkout_path = checkout_path_patch.start()
self.addCleanup(checkout_path_patch.stop)
self.project = ProjectFactory.create()
self.vcs_project = VCSProject(self.project)
class VCSProjectTests(TestCase):
def setUp(self):
# Force the checkout path to point to a test directory to make
# resource file loading pass during tests.
checkout_path_patch = patch.object(
Project,
'checkout_path',
new_callable=PropertyMock,
return_value=os.path.join(TEST_CHECKOUT_PATH, 'no_resources_test')
)
self.mock_checkout_path = checkout_path_patch.start()
self.addCleanup(checkout_path_patch.stop)
self.project = ProjectFactory.create()
self.vcs_project = VCSProject(self.project)
def test_relative_resource_paths(self):
self.vcs_project.source_directory_path = Mock(return_value='/root/')
self.vcs_project.resources_for_path = Mock(return_value=[
'/root/foo.po',
'/root/meh/bar.po'
])
assert_equal(
list(self.vcs_project.relative_resource_paths()),
['foo.po', 'meh/bar.po']
)
def test_relative_resource_paths_pot(self):
"""
If a resource ends in .pot, replace the extension with .po since
relative paths are used within non-source locales that do not
have .pot files.
"""
self.vcs_project.source_directory_path = Mock(return_value='/root/')
self.vcs_project.resources_for_path = Mock(return_value=[
'/root/foo.pot',
'/root/meh/bar.pot'
])
assert_equal(
list(self.vcs_project.relative_resource_paths()),
['foo.po', 'meh/bar.po']
)
def test_source_directory_path_no_resource(self):
"""
When searching for source directories, do not match directories that
do not contain resource files.
"""
checkout_path = os.path.join(TEST_CHECKOUT_PATH, 'no_resources_test')
self.mock_checkout_path.return_value = checkout_path
assert_equal(
self.vcs_project.source_directory_path(),
os.path.join(checkout_path, 'real_resources', 'templates')
)
def test_source_directory_scoring_templates(self):
"""
When searching for source directories, prefer directories named
`templates` over all others.
"""
checkout_path = os.path.join(TEST_CHECKOUT_PATH, 'scoring_templates_test')
self.mock_checkout_path.return_value = checkout_path
assert_equal(
self.vcs_project.source_directory_path(),
os.path.join(checkout_path, 'templates')
)
def test_source_directory_scoring_en_US(self):
"""
When searching for source directories, prefer directories named
`en-US` over others besides `templates`.
"""
checkout_path = os.path.join(TEST_CHECKOUT_PATH, 'scoring_en_US_test')
self.mock_checkout_path.return_value = checkout_path
assert_equal(
self.vcs_project.source_directory_path(),
os.path.join(checkout_path, 'en-US')
)
def test_source_directory_scoring_source_files(self):
"""
When searching for source directories, prefer directories with
source-only formats over all others.
"""
checkout_path = os.path.join(TEST_CHECKOUT_PATH, 'scoring_source_files_test')
self.mock_checkout_path.return_value = checkout_path
assert_equal(
self.vcs_project.source_directory_path(),
os.path.join(checkout_path, 'en') # en has pot files in it
)
def test_resources_parse_error(self):
"""
If VCSResource() raises a ParseError while loading, log an error
and skip the resource.
"""
self.vcs_project.relative_resource_paths = Mock(return_value=['failure', 'success'])
# Fail only if the path is failure so we can test the ignore.
def vcs_resource_constructor(project, path, locales=None):
if path == 'failure':
raise ParseError('error message')
else:
return 'successful resource'
with patch('pontoon.sync.vcs_models.VCSResource') as MockVCSResource, \
patch('pontoon.sync.vcs_models.log') as mock_log:
MockVCSResource.side_effect = vcs_resource_constructor
assert_equal(self.vcs_project.resources, {'success': 'successful resource'})
mock_log.error.assert_called_with(CONTAINS('failure', 'error message'))
def test_resource_for_path_region_properties(self):
"""
If a project has a repository_url in pontoon.base.MOZILLA_REPOS,
resources_for_path should ignore files named
"region.properties".
"""
url = 'https://moz.example.com'
self.project.repositories.all().delete()
self.project.repositories.add(RepositoryFactory.build(url=url))
with patch('pontoon.sync.vcs_models.os', wraps=os) as mock_os, \
patch('pontoon.sync.vcs_models.MOZILLA_REPOS', [url]):
mock_os.walk.return_value = [
('/root', [], ['foo.pot', 'region.properties'])
]
assert_equal(
list(self.vcs_project.resources_for_path('/root')),
[os.path.join('/root', 'foo.pot')]
)
def sync_project(db_project, no_pull=False, no_commit=False):
"""
Update the database with the current state of resources in version
control and write any submitted translations from the database back
to version control.
"""
# Mark "now" at the start of sync to avoid messing with
# translations submitted during sync.
now = timezone.now()
# Pull changes from VCS and update what we know about the files.
if not no_pull:
repos_changed = pull_changes(db_project)
else:
repos_changed = True # Assume changed.
# If the repos haven't changed since the last sync and there are
# no Pontoon-side changes for this project, quit early.
if not repos_changed and not db_project.needs_sync:
log.info('Skipping project {0}, no changes detected.'.format(db_project.slug))
return
vcs_project = VCSProject(db_project)
update_resources(db_project, vcs_project)
# Collect all entities across VCS and the database and get their
# keys so we can match up matching entities.
vcs_entities = get_vcs_entities(vcs_project)
db_entities = get_db_entities(db_project)
entity_keys = set().union(db_entities.keys(), vcs_entities.keys())
changeset = ChangeSet(db_project, vcs_project, now)
for key in entity_keys:
db_entity = db_entities.get(key, None)
vcs_entity = vcs_entities.get(key, None)
handle_entity(changeset, db_project, key, db_entity, vcs_entity)
# Apply the changeset to the files, commit them, and update stats
# entries in the DB.
changeset.execute()
if not no_commit:
commit_changes(db_project, vcs_project, changeset)
update_project_stats(db_project, vcs_project, changeset)
# Clear out the "has_changed" markers now that we've finished
# syncing.
(ChangedEntityLocale.objects
.filter(entity__resource__project=db_project, when__lte=now)
.delete())
db_project.has_changed = False
db_project.save()
# Clean up any duplicate approvals at the end of sync right
# before we commit the transaction to avoid race conditions.
with connection.cursor() as cursor:
cursor.execute("""
UPDATE base_translation AS b
SET approved = FALSE, approved_date = NULL
WHERE approved_date !=
(SELECT max(approved_date)
FROM base_translation
WHERE entity_id = b.entity_id
AND locale_id = b.locale_id
AND (plural_form = b.plural_form OR plural_form IS NULL));
""")
log.info(u'Synced project {0}'.format(db_project.slug))
class VCSProjectTests(TestCase):
def setUp(self):
# Force the checkout path to point to a test directory to make
# resource file loading pass during tests.
checkout_path_patch = patch.object(Project,
'checkout_path',
new_callable=PropertyMock,
return_value=os.path.join(
TEST_CHECKOUT_PATH,
'no_resources_test'))
self.mock_checkout_path = checkout_path_patch.start()
self.addCleanup(checkout_path_patch.stop)
self.project = ProjectFactory.create()
self.vcs_project = VCSProject(self.project)
def test_relative_resource_paths(self):
self.vcs_project.source_directory_path = Mock(return_value='/root/')
self.vcs_project.resources_for_path = Mock(
return_value=['/root/foo.po', '/root/meh/bar.po'])
assert_equal(list(self.vcs_project.relative_resource_paths()),
['foo.po', 'meh/bar.po'])
def test_relative_resource_paths_pot(self):
"""
If a resource ends in .pot, replace the extension with .po since
relative paths are used within non-source locales that do not
have .pot files.
"""
self.vcs_project.source_directory_path = Mock(return_value='/root/')
self.vcs_project.resources_for_path = Mock(
return_value=['/root/foo.pot', '/root/meh/bar.pot'])
assert_equal(list(self.vcs_project.relative_resource_paths()),
['foo.po', 'meh/bar.po'])
def test_source_directory_path_no_resource(self):
"""
When searching for source directories, do not match directories that
do not contain resource files.
"""
checkout_path = os.path.join(TEST_CHECKOUT_PATH, 'no_resources_test')
self.mock_checkout_path.return_value = checkout_path
assert_equal(
self.vcs_project.source_directory_path(),
os.path.join(checkout_path, 'real_resources', 'templates'))
def test_source_directory_scoring_templates(self):
"""
When searching for source directories, prefer directories named
`templates` over all others.
"""
checkout_path = os.path.join(TEST_CHECKOUT_PATH,
'scoring_templates_test')
self.mock_checkout_path.return_value = checkout_path
assert_equal(self.vcs_project.source_directory_path(),
os.path.join(checkout_path, 'templates'))
def test_source_directory_scoring_en_US(self):
"""
When searching for source directories, prefer directories named
`en-US` over others besides `templates`.
"""
checkout_path = os.path.join(TEST_CHECKOUT_PATH, 'scoring_en_US_test')
self.mock_checkout_path.return_value = checkout_path
assert_equal(self.vcs_project.source_directory_path(),
os.path.join(checkout_path, 'en-US'))
def test_source_directory_scoring_source_files(self):
"""
When searching for source directories, prefer directories with
source-only formats over all others.
"""
checkout_path = os.path.join(TEST_CHECKOUT_PATH,
'scoring_source_files_test')
self.mock_checkout_path.return_value = checkout_path
assert_equal(
self.vcs_project.source_directory_path(),
os.path.join(checkout_path, 'en') # en has pot files in it
)
def test_resources_parse_error(self):
"""
If VCSResource() raises a ParseError while loading, log an error
and skip the resource.
"""
self.vcs_project.relative_resource_paths = Mock(
return_value=['failure', 'success'])
# Fail only if the path is failure so we can test the ignore.
def vcs_resource_constructor(project, path):
if path == 'failure':
raise ParseError('error message')
else:
return 'successful resource'
with patch('pontoon.sync.vcs_models.VCSResource') as MockVCSResource, \
patch('pontoon.sync.vcs_models.log') as mock_log:
MockVCSResource.side_effect = vcs_resource_constructor
assert_equal(self.vcs_project.resources,
{'success': 'successful resource'})
mock_log.error.assert_called_with(
CONTAINS('failure', 'error message'))
def test_resource_for_path_region_properties(self):
"""
If a project has a repository_url in pontoon.base.MOZILLA_REPOS,
resources_for_path should ignore files named
"region.properties".
"""
url = 'https://moz.example.com'
self.project.repositories.all().delete()
self.project.repositories.add(RepositoryFactory.build(url=url))
with patch('pontoon.sync.vcs_models.os', wraps=os) as mock_os, \
patch('pontoon.sync.vcs_models.MOZILLA_REPOS', [url]):
mock_os.walk.return_value = [('/root', [],
['foo.pot', 'region.properties'])]
assert_equal(list(self.vcs_project.resources_for_path('/root')),
[os.path.join('/root', 'foo.pot')])
def sync_project_repo(self, project_pk, repo_pk, project_sync_log_pk, now,
no_pull=False, no_commit=False):
db_project = get_or_fail(Project, pk=project_pk,
message='Could not sync project with pk={0}, not found.'.format(project_pk))
repo = get_or_fail(Repository, pk=repo_pk,
message='Could not sync repo with pk={0}, not found.'.format(project_pk))
project_sync_log = get_or_fail(ProjectSyncLog, pk=project_sync_log_pk,
message=('Could not sync project {0}, log with pk={1} not found.'
.format(db_project.slug, project_sync_log_pk)))
repo_sync_log = RepositorySyncLog.objects.create(
project_sync_log=project_sync_log,
repository=repo,
start_time=timezone.now()
)
# Pull VCS changes in case we're on a different worker than the one
# sync started on.
if not no_pull:
pull_changes(db_project)
if len(repo.locales) < 1:
log.warning('Could not sync repo `{0}`, no locales found within.'
.format(repo.url))
repo_sync_log.end_time = timezone.now()
repo_sync_log.save(update_fields=['end_time'])
return
vcs_project = VCSProject(db_project, locales=repo.locales)
for locale in repo.locales:
try:
with transaction.atomic():
changeset = ChangeSet(db_project, vcs_project, now)
update_translations(db_project, vcs_project, locale, changeset)
changeset.execute()
update_project_stats(db_project, vcs_project, changeset, locale)
# Clear out the "has_changed" markers now that we've finished
# syncing.
(ChangedEntityLocale.objects
.filter(entity__resource__project=db_project,
locale=locale,
when__lte=now)
.delete())
db_project.has_changed = False
db_project.save(update_fields=['has_changed'])
# Clean up any duplicate approvals at the end of sync right
# before we commit the transaction to avoid race conditions.
with connection.cursor() as cursor:
cursor.execute("""
UPDATE base_translation AS b
SET approved = FALSE, approved_date = NULL
WHERE
id IN
(SELECT trans.id FROM base_translation AS trans
LEFT JOIN base_entity AS ent ON ent.id = trans.entity_id
LEFT JOIN base_resource AS res ON res.id = ent.resource_id
WHERE locale_id = %(locale_id)s
AND res.project_id = %(project_id)s)
AND approved_date !=
(SELECT max(approved_date)
FROM base_translation
WHERE entity_id = b.entity_id
AND locale_id = b.locale_id
AND (plural_form = b.plural_form OR plural_form IS NULL));
""", {
'locale_id': locale.id,
'project_id': db_project.id
})
# Perform the commit last so that, if it succeeds, there is
# nothing after it to fail.
if not no_commit and locale in changeset.locales_to_commit:
commit_changes(db_project, vcs_project, changeset, locale)
except CommitToRepositoryException as err:
# Transaction aborted, log and move on to the next locale.
log.warning(
'Failed to sync locale {locale} for project {project} due to '
'commit error: {error}'.format(
locale=locale.code,
project=db_project.slug,
error=err,
)
)
repo_sync_log.end_time = timezone.now()
repo_sync_log.save()
log.info('Synced translations for project {0} in locales {1}.'.format(
db_project.slug, ','.join(locale.code for locale in repo.locales)
))
def handle_upload_content(slug, code, part, f, user):
"""
Update translations in the database from uploaded file.
:param str slug: Project slug.
:param str code: Locale code.
:param str part: Resource path or Subpage name.
:param UploadedFile f: UploadedFile instance.
:param User user: User uploading the file.
"""
# Avoid circular import; someday we should refactor to avoid.
from pontoon.sync import formats
from pontoon.sync.changeset import ChangeSet
from pontoon.sync.vcs_models import VCSProject
from pontoon.base.models import (
ChangedEntityLocale,
Entity,
Locale,
Project,
Resource,
Translation,
update_stats,
)
relative_path = _get_relative_path_from_part(slug, part)
project = get_object_or_404(Project, slug=slug)
locale = get_object_or_404(Locale, code__iexact=code)
resource = get_object_or_404(Resource,
project__slug=slug,
path=relative_path)
# Store uploaded file to a temporary file and parse it
extension = os.path.splitext(f.name)[1]
with tempfile.NamedTemporaryFile(suffix=extension) as temp:
for chunk in f.chunks():
temp.write(chunk)
temp.flush()
resource_file = formats.parse(temp.name)
# Update database objects from file
changeset = ChangeSet(project, VCSProject(project, locales=[locale]),
timezone.now())
entities_qs = Entity.objects.filter(
resource__project=project,
resource__path=relative_path,
obsolete=False).prefetch_related(
Prefetch('translation_set',
queryset=Translation.objects.filter(locale=locale),
to_attr='db_translations')).prefetch_related(
Prefetch('translation_set',
queryset=Translation.objects.filter(
locale=locale,
approved_date__lte=timezone.now()),
to_attr='old_translations'))
entities_dict = {entity.key: entity for entity in entities_qs}
for vcs_translation in resource_file.translations:
key = vcs_translation.key
if key in entities_dict:
entity = entities_dict[key]
changeset.update_entity_translations_from_vcs(
entity, locale.code, vcs_translation, user,
entity.db_translations, entity.old_translations)
changeset.bulk_create_translations()
changeset.bulk_update_translations()
update_stats(resource, locale)
# Mark translations as changed
changed_entities = {}
existing = ChangedEntityLocale.objects.values_list('entity',
'locale').distinct()
for t in changeset.translations_to_create + changeset.translations_to_update:
key = (t.entity.pk, t.locale.pk)
# Remove duplicate changes to prevent unique constraint violation
if not key in existing:
changed_entities[key] = ChangedEntityLocale(entity=t.entity,
locale=t.locale)
ChangedEntityLocale.objects.bulk_create(changed_entities.values())
def setUp(self):
self.now = aware_datetime(1970, 1, 1)
timezone_patch = patch('pontoon.sync.tasks.timezone')
self.mock_timezone = timezone_patch.start()
self.addCleanup(timezone_patch.stop)
self.mock_timezone.now.return_value = self.now
self.translated_locale = LocaleFactory.create(code='translated-locale')
self.inactive_locale = LocaleFactory.create(code='inactive-locale')
self.repository = RepositoryFactory()
self.db_project = ProjectFactory.create(
name='db-project',
locales=[self.translated_locale],
repositories=[self.repository]
)
self.main_db_resource = ResourceFactory.create(
project=self.db_project,
path='main.lang',
format='lang'
)
self.other_db_resource = ResourceFactory.create(
project=self.db_project,
path='other.lang',
format='lang'
)
self.missing_db_resource = ResourceFactory.create(
project=self.db_project,
path='missing.lang',
format='lang'
)
self.main_db_entity = EntityFactory.create(
resource=self.main_db_resource,
string='Source String',
key='Source String',
obsolete=False
)
self.other_db_entity = EntityFactory.create(
resource=self.other_db_resource,
string='Other Source String',
key='Other Source String',
obsolete=False
)
self.main_db_translation = TranslationFactory.create(
entity=self.main_db_entity,
plural_form=None,
locale=self.translated_locale,
string='Translated String',
date=aware_datetime(1970, 1, 1),
approved=True,
extra={'tags': []}
)
# Load paths from the fake locale directory.
checkout_path_patch = patch.object(
Project,
'checkout_path',
new_callable=PropertyMock,
return_value=FAKE_CHECKOUT_PATH
)
checkout_path_patch.start()
self.addCleanup(checkout_path_patch.stop)
self.vcs_project = VCSProject(self.db_project)
self.main_vcs_resource = self.vcs_project.resources[self.main_db_resource.path]
self.other_vcs_resource = self.vcs_project.resources[self.other_db_resource.path]
self.missing_vcs_resource = self.vcs_project.resources[self.missing_db_resource.path]
self.main_vcs_entity = self.main_vcs_resource.entities['Source String']
self.main_vcs_translation = self.main_vcs_entity.translations['translated-locale']
# Mock VCSResource.save() for each resource to avoid altering
# the filesystem.
resource_save_patch = patch.object(VCSResource, 'save')
resource_save_patch.start()
self.addCleanup(resource_save_patch.stop)
self.changeset = ChangeSet(
self.db_project,
self.vcs_project,
aware_datetime(1970, 1, 1)
)
