def setUp(self): # Force the checkout path to point to a test directory to make # resource file loading pass during tests. checkout_path_patch = patch.object(Project, 'checkout_path', new_callable=PropertyMock, return_value=os.path.join( TEST_CHECKOUT_PATH, 'no_resources_test')) self.mock_checkout_path = checkout_path_patch.start() self.addCleanup(checkout_path_patch.stop) self.project = ProjectFactory.create() self.vcs_project = VCSProject(self.project)
def sync_project(db_project, now): # Only load source resources for updating entities. vcs_project = VCSProject(db_project, locales=[]) with transaction.atomic(): update_resources(db_project, vcs_project) changeset = ChangeSet(db_project, vcs_project, now) update_entities(db_project, vcs_project, changeset) changeset.execute()
def setUp(self): # Force the checkout path to point to a test directory to make # resource file loading pass during tests. checkout_path_patch = patch.object( Project, 'checkout_path', new_callable=PropertyMock, return_value=os.path.join(TEST_CHECKOUT_PATH, 'no_resources_test') ) self.mock_checkout_path = checkout_path_patch.start() self.addCleanup(checkout_path_patch.stop) self.project = ProjectFactory.create() self.vcs_project = VCSProject(self.project)
class VCSProjectTests(TestCase): def setUp(self): # Force the checkout path to point to a test directory to make # resource file loading pass during tests. checkout_path_patch = patch.object( Project, 'checkout_path', new_callable=PropertyMock, return_value=os.path.join(TEST_CHECKOUT_PATH, 'no_resources_test') ) self.mock_checkout_path = checkout_path_patch.start() self.addCleanup(checkout_path_patch.stop) self.project = ProjectFactory.create() self.vcs_project = VCSProject(self.project) def test_relative_resource_paths(self): self.vcs_project.source_directory_path = Mock(return_value='/root/') self.vcs_project.resources_for_path = Mock(return_value=[ '/root/foo.po', '/root/meh/bar.po' ]) assert_equal( list(self.vcs_project.relative_resource_paths()), ['foo.po', 'meh/bar.po'] ) def test_relative_resource_paths_pot(self): """ If a resource ends in .pot, replace the extension with .po since relative paths are used within non-source locales that do not have .pot files. """ self.vcs_project.source_directory_path = Mock(return_value='/root/') self.vcs_project.resources_for_path = Mock(return_value=[ '/root/foo.pot', '/root/meh/bar.pot' ]) assert_equal( list(self.vcs_project.relative_resource_paths()), ['foo.po', 'meh/bar.po'] ) def test_source_directory_path_no_resource(self): """ When searching for source directories, do not match directories that do not contain resource files. """ checkout_path = os.path.join(TEST_CHECKOUT_PATH, 'no_resources_test') self.mock_checkout_path.return_value = checkout_path assert_equal( self.vcs_project.source_directory_path(), os.path.join(checkout_path, 'real_resources', 'templates') ) def test_source_directory_scoring_templates(self): """ When searching for source directories, prefer directories named `templates` over all others. """ checkout_path = os.path.join(TEST_CHECKOUT_PATH, 'scoring_templates_test') self.mock_checkout_path.return_value = checkout_path assert_equal( self.vcs_project.source_directory_path(), os.path.join(checkout_path, 'templates') ) def test_source_directory_scoring_en_US(self): """ When searching for source directories, prefer directories named `en-US` over others besides `templates`. """ checkout_path = os.path.join(TEST_CHECKOUT_PATH, 'scoring_en_US_test') self.mock_checkout_path.return_value = checkout_path assert_equal( self.vcs_project.source_directory_path(), os.path.join(checkout_path, 'en-US') ) def test_source_directory_scoring_source_files(self): """ When searching for source directories, prefer directories with source-only formats over all others. """ checkout_path = os.path.join(TEST_CHECKOUT_PATH, 'scoring_source_files_test') self.mock_checkout_path.return_value = checkout_path assert_equal( self.vcs_project.source_directory_path(), os.path.join(checkout_path, 'en') # en has pot files in it ) def test_resources_parse_error(self): """ If VCSResource() raises a ParseError while loading, log an error and skip the resource. """ self.vcs_project.relative_resource_paths = Mock(return_value=['failure', 'success']) # Fail only if the path is failure so we can test the ignore. def vcs_resource_constructor(project, path, locales=None): if path == 'failure': raise ParseError('error message') else: return 'successful resource' with patch('pontoon.sync.vcs_models.VCSResource') as MockVCSResource, \ patch('pontoon.sync.vcs_models.log') as mock_log: MockVCSResource.side_effect = vcs_resource_constructor assert_equal(self.vcs_project.resources, {'success': 'successful resource'}) mock_log.error.assert_called_with(CONTAINS('failure', 'error message')) def test_resource_for_path_region_properties(self): """ If a project has a repository_url in pontoon.base.MOZILLA_REPOS, resources_for_path should ignore files named "region.properties". """ url = 'https://moz.example.com' self.project.repositories.all().delete() self.project.repositories.add(RepositoryFactory.build(url=url)) with patch('pontoon.sync.vcs_models.os', wraps=os) as mock_os, \ patch('pontoon.sync.vcs_models.MOZILLA_REPOS', [url]): mock_os.walk.return_value = [ ('/root', [], ['foo.pot', 'region.properties']) ] assert_equal( list(self.vcs_project.resources_for_path('/root')), [os.path.join('/root', 'foo.pot')] )
def sync_project(db_project, no_pull=False, no_commit=False): """ Update the database with the current state of resources in version control and write any submitted translations from the database back to version control. """ # Mark "now" at the start of sync to avoid messing with # translations submitted during sync. now = timezone.now() # Pull changes from VCS and update what we know about the files. if not no_pull: repos_changed = pull_changes(db_project) else: repos_changed = True # Assume changed. # If the repos haven't changed since the last sync and there are # no Pontoon-side changes for this project, quit early. if not repos_changed and not db_project.needs_sync: log.info('Skipping project {0}, no changes detected.'.format(db_project.slug)) return vcs_project = VCSProject(db_project) update_resources(db_project, vcs_project) # Collect all entities across VCS and the database and get their # keys so we can match up matching entities. vcs_entities = get_vcs_entities(vcs_project) db_entities = get_db_entities(db_project) entity_keys = set().union(db_entities.keys(), vcs_entities.keys()) changeset = ChangeSet(db_project, vcs_project, now) for key in entity_keys: db_entity = db_entities.get(key, None) vcs_entity = vcs_entities.get(key, None) handle_entity(changeset, db_project, key, db_entity, vcs_entity) # Apply the changeset to the files, commit them, and update stats # entries in the DB. changeset.execute() if not no_commit: commit_changes(db_project, vcs_project, changeset) update_project_stats(db_project, vcs_project, changeset) # Clear out the "has_changed" markers now that we've finished # syncing. (ChangedEntityLocale.objects .filter(entity__resource__project=db_project, when__lte=now) .delete()) db_project.has_changed = False db_project.save() # Clean up any duplicate approvals at the end of sync right # before we commit the transaction to avoid race conditions. with connection.cursor() as cursor: cursor.execute(""" UPDATE base_translation AS b SET approved = FALSE, approved_date = NULL WHERE approved_date != (SELECT max(approved_date) FROM base_translation WHERE entity_id = b.entity_id AND locale_id = b.locale_id AND (plural_form = b.plural_form OR plural_form IS NULL)); """) log.info(u'Synced project {0}'.format(db_project.slug))
class VCSProjectTests(TestCase): def setUp(self): # Force the checkout path to point to a test directory to make # resource file loading pass during tests. checkout_path_patch = patch.object(Project, 'checkout_path', new_callable=PropertyMock, return_value=os.path.join( TEST_CHECKOUT_PATH, 'no_resources_test')) self.mock_checkout_path = checkout_path_patch.start() self.addCleanup(checkout_path_patch.stop) self.project = ProjectFactory.create() self.vcs_project = VCSProject(self.project) def test_relative_resource_paths(self): self.vcs_project.source_directory_path = Mock(return_value='/root/') self.vcs_project.resources_for_path = Mock( return_value=['/root/foo.po', '/root/meh/bar.po']) assert_equal(list(self.vcs_project.relative_resource_paths()), ['foo.po', 'meh/bar.po']) def test_relative_resource_paths_pot(self): """ If a resource ends in .pot, replace the extension with .po since relative paths are used within non-source locales that do not have .pot files. """ self.vcs_project.source_directory_path = Mock(return_value='/root/') self.vcs_project.resources_for_path = Mock( return_value=['/root/foo.pot', '/root/meh/bar.pot']) assert_equal(list(self.vcs_project.relative_resource_paths()), ['foo.po', 'meh/bar.po']) def test_source_directory_path_no_resource(self): """ When searching for source directories, do not match directories that do not contain resource files. """ checkout_path = os.path.join(TEST_CHECKOUT_PATH, 'no_resources_test') self.mock_checkout_path.return_value = checkout_path assert_equal( self.vcs_project.source_directory_path(), os.path.join(checkout_path, 'real_resources', 'templates')) def test_source_directory_scoring_templates(self): """ When searching for source directories, prefer directories named `templates` over all others. """ checkout_path = os.path.join(TEST_CHECKOUT_PATH, 'scoring_templates_test') self.mock_checkout_path.return_value = checkout_path assert_equal(self.vcs_project.source_directory_path(), os.path.join(checkout_path, 'templates')) def test_source_directory_scoring_en_US(self): """ When searching for source directories, prefer directories named `en-US` over others besides `templates`. """ checkout_path = os.path.join(TEST_CHECKOUT_PATH, 'scoring_en_US_test') self.mock_checkout_path.return_value = checkout_path assert_equal(self.vcs_project.source_directory_path(), os.path.join(checkout_path, 'en-US')) def test_source_directory_scoring_source_files(self): """ When searching for source directories, prefer directories with source-only formats over all others. """ checkout_path = os.path.join(TEST_CHECKOUT_PATH, 'scoring_source_files_test') self.mock_checkout_path.return_value = checkout_path assert_equal( self.vcs_project.source_directory_path(), os.path.join(checkout_path, 'en') # en has pot files in it ) def test_resources_parse_error(self): """ If VCSResource() raises a ParseError while loading, log an error and skip the resource. """ self.vcs_project.relative_resource_paths = Mock( return_value=['failure', 'success']) # Fail only if the path is failure so we can test the ignore. def vcs_resource_constructor(project, path): if path == 'failure': raise ParseError('error message') else: return 'successful resource' with patch('pontoon.sync.vcs_models.VCSResource') as MockVCSResource, \ patch('pontoon.sync.vcs_models.log') as mock_log: MockVCSResource.side_effect = vcs_resource_constructor assert_equal(self.vcs_project.resources, {'success': 'successful resource'}) mock_log.error.assert_called_with( CONTAINS('failure', 'error message')) def test_resource_for_path_region_properties(self): """ If a project has a repository_url in pontoon.base.MOZILLA_REPOS, resources_for_path should ignore files named "region.properties". """ url = 'https://moz.example.com' self.project.repositories.all().delete() self.project.repositories.add(RepositoryFactory.build(url=url)) with patch('pontoon.sync.vcs_models.os', wraps=os) as mock_os, \ patch('pontoon.sync.vcs_models.MOZILLA_REPOS', [url]): mock_os.walk.return_value = [('/root', [], ['foo.pot', 'region.properties'])] assert_equal(list(self.vcs_project.resources_for_path('/root')), [os.path.join('/root', 'foo.pot')])
def sync_project_repo(self, project_pk, repo_pk, project_sync_log_pk, now, no_pull=False, no_commit=False): db_project = get_or_fail(Project, pk=project_pk, message='Could not sync project with pk={0}, not found.'.format(project_pk)) repo = get_or_fail(Repository, pk=repo_pk, message='Could not sync repo with pk={0}, not found.'.format(project_pk)) project_sync_log = get_or_fail(ProjectSyncLog, pk=project_sync_log_pk, message=('Could not sync project {0}, log with pk={1} not found.' .format(db_project.slug, project_sync_log_pk))) repo_sync_log = RepositorySyncLog.objects.create( project_sync_log=project_sync_log, repository=repo, start_time=timezone.now() ) # Pull VCS changes in case we're on a different worker than the one # sync started on. if not no_pull: pull_changes(db_project) if len(repo.locales) < 1: log.warning('Could not sync repo `{0}`, no locales found within.' .format(repo.url)) repo_sync_log.end_time = timezone.now() repo_sync_log.save(update_fields=['end_time']) return vcs_project = VCSProject(db_project, locales=repo.locales) for locale in repo.locales: try: with transaction.atomic(): changeset = ChangeSet(db_project, vcs_project, now) update_translations(db_project, vcs_project, locale, changeset) changeset.execute() update_project_stats(db_project, vcs_project, changeset, locale) # Clear out the "has_changed" markers now that we've finished # syncing. (ChangedEntityLocale.objects .filter(entity__resource__project=db_project, locale=locale, when__lte=now) .delete()) db_project.has_changed = False db_project.save(update_fields=['has_changed']) # Clean up any duplicate approvals at the end of sync right # before we commit the transaction to avoid race conditions. with connection.cursor() as cursor: cursor.execute(""" UPDATE base_translation AS b SET approved = FALSE, approved_date = NULL WHERE id IN (SELECT trans.id FROM base_translation AS trans LEFT JOIN base_entity AS ent ON ent.id = trans.entity_id LEFT JOIN base_resource AS res ON res.id = ent.resource_id WHERE locale_id = %(locale_id)s AND res.project_id = %(project_id)s) AND approved_date != (SELECT max(approved_date) FROM base_translation WHERE entity_id = b.entity_id AND locale_id = b.locale_id AND (plural_form = b.plural_form OR plural_form IS NULL)); """, { 'locale_id': locale.id, 'project_id': db_project.id }) # Perform the commit last so that, if it succeeds, there is # nothing after it to fail. if not no_commit and locale in changeset.locales_to_commit: commit_changes(db_project, vcs_project, changeset, locale) except CommitToRepositoryException as err: # Transaction aborted, log and move on to the next locale. log.warning( 'Failed to sync locale {locale} for project {project} due to ' 'commit error: {error}'.format( locale=locale.code, project=db_project.slug, error=err, ) ) repo_sync_log.end_time = timezone.now() repo_sync_log.save() log.info('Synced translations for project {0} in locales {1}.'.format( db_project.slug, ','.join(locale.code for locale in repo.locales) ))
def handle_upload_content(slug, code, part, f, user): """ Update translations in the database from uploaded file. :param str slug: Project slug. :param str code: Locale code. :param str part: Resource path or Subpage name. :param UploadedFile f: UploadedFile instance. :param User user: User uploading the file. """ # Avoid circular import; someday we should refactor to avoid. from pontoon.sync import formats from pontoon.sync.changeset import ChangeSet from pontoon.sync.vcs_models import VCSProject from pontoon.base.models import ( ChangedEntityLocale, Entity, Locale, Project, Resource, Translation, update_stats, ) relative_path = _get_relative_path_from_part(slug, part) project = get_object_or_404(Project, slug=slug) locale = get_object_or_404(Locale, code__iexact=code) resource = get_object_or_404(Resource, project__slug=slug, path=relative_path) # Store uploaded file to a temporary file and parse it extension = os.path.splitext(f.name)[1] with tempfile.NamedTemporaryFile(suffix=extension) as temp: for chunk in f.chunks(): temp.write(chunk) temp.flush() resource_file = formats.parse(temp.name) # Update database objects from file changeset = ChangeSet(project, VCSProject(project, locales=[locale]), timezone.now()) entities_qs = Entity.objects.filter( resource__project=project, resource__path=relative_path, obsolete=False).prefetch_related( Prefetch('translation_set', queryset=Translation.objects.filter(locale=locale), to_attr='db_translations')).prefetch_related( Prefetch('translation_set', queryset=Translation.objects.filter( locale=locale, approved_date__lte=timezone.now()), to_attr='old_translations')) entities_dict = {entity.key: entity for entity in entities_qs} for vcs_translation in resource_file.translations: key = vcs_translation.key if key in entities_dict: entity = entities_dict[key] changeset.update_entity_translations_from_vcs( entity, locale.code, vcs_translation, user, entity.db_translations, entity.old_translations) changeset.bulk_create_translations() changeset.bulk_update_translations() update_stats(resource, locale) # Mark translations as changed changed_entities = {} existing = ChangedEntityLocale.objects.values_list('entity', 'locale').distinct() for t in changeset.translations_to_create + changeset.translations_to_update: key = (t.entity.pk, t.locale.pk) # Remove duplicate changes to prevent unique constraint violation if not key in existing: changed_entities[key] = ChangedEntityLocale(entity=t.entity, locale=t.locale) ChangedEntityLocale.objects.bulk_create(changed_entities.values())
def setUp(self): self.now = aware_datetime(1970, 1, 1) timezone_patch = patch('pontoon.sync.tasks.timezone') self.mock_timezone = timezone_patch.start() self.addCleanup(timezone_patch.stop) self.mock_timezone.now.return_value = self.now self.translated_locale = LocaleFactory.create(code='translated-locale') self.inactive_locale = LocaleFactory.create(code='inactive-locale') self.repository = RepositoryFactory() self.db_project = ProjectFactory.create( name='db-project', locales=[self.translated_locale], repositories=[self.repository] ) self.main_db_resource = ResourceFactory.create( project=self.db_project, path='main.lang', format='lang' ) self.other_db_resource = ResourceFactory.create( project=self.db_project, path='other.lang', format='lang' ) self.missing_db_resource = ResourceFactory.create( project=self.db_project, path='missing.lang', format='lang' ) self.main_db_entity = EntityFactory.create( resource=self.main_db_resource, string='Source String', key='Source String', obsolete=False ) self.other_db_entity = EntityFactory.create( resource=self.other_db_resource, string='Other Source String', key='Other Source String', obsolete=False ) self.main_db_translation = TranslationFactory.create( entity=self.main_db_entity, plural_form=None, locale=self.translated_locale, string='Translated String', date=aware_datetime(1970, 1, 1), approved=True, extra={'tags': []} ) # Load paths from the fake locale directory. checkout_path_patch = patch.object( Project, 'checkout_path', new_callable=PropertyMock, return_value=FAKE_CHECKOUT_PATH ) checkout_path_patch.start() self.addCleanup(checkout_path_patch.stop) self.vcs_project = VCSProject(self.db_project) self.main_vcs_resource = self.vcs_project.resources[self.main_db_resource.path] self.other_vcs_resource = self.vcs_project.resources[self.other_db_resource.path] self.missing_vcs_resource = self.vcs_project.resources[self.missing_db_resource.path] self.main_vcs_entity = self.main_vcs_resource.entities['Source String'] self.main_vcs_translation = self.main_vcs_entity.translations['translated-locale'] # Mock VCSResource.save() for each resource to avoid altering # the filesystem. resource_save_patch = patch.object(VCSResource, 'save') resource_save_patch.start() self.addCleanup(resource_save_patch.stop) self.changeset = ChangeSet( self.db_project, self.vcs_project, aware_datetime(1970, 1, 1) )