def setUp(self): # Force the checkout path to point to a test directory to make # resource file loading pass during tests. checkout_path_patch = patch.object(Project, 'checkout_path', new_callable=PropertyMock, return_value=os.path.join( TEST_CHECKOUT_PATH, 'no_resources_test')) self.mock_checkout_path = checkout_path_patch.start() self.addCleanup(checkout_path_patch.stop) self.project = ProjectFactory.create() self.vcs_project = VCSProject(self.project)
def setUp(self): self.locale, _ = Locale.objects.get_or_create(code='fr') self.repository = RepositoryFactory() self.db_project = ProjectFactory.create( repositories=[self.repository], ) checkout_path_patch = patch.object( Repository, 'checkout_path', new_callable=PropertyMock, return_value=PROJECT_CONFIG_CHECKOUT_PATH ) self.mock_checkout_path = checkout_path_patch.start() self.addCleanup(checkout_path_patch.stop) self.resource_strings = ResourceFactory.create( project=self.db_project, path='values/strings.properties', ) self.resource_strings_reality = ResourceFactory.create( project=self.db_project, path='values/strings_reality.properties', ) # Make sure VCSConfiguration instance is initialized self.db_project.configuration_file = 'l10n.toml' self.vcs_project = VCSProject(self.db_project) self.vcs_project.configuration.configuration_path = os.path.join( PROJECT_CONFIG_CHECKOUT_PATH, self.db_project.configuration_file, )
def sync_project(db_project, now): # Only load source resources for updating entities. vcs_project = VCSProject(db_project, locales=[]) with transaction.atomic(): update_resources(db_project, vcs_project) changeset = ChangeSet(db_project, vcs_project, now) update_entities(db_project, vcs_project, changeset) changeset.execute()
def sync_project(db_project, now, full_scan=False): vcs_project = VCSProject(db_project, locales=[], full_scan=full_scan) with transaction.atomic(): removed_paths = update_resources(db_project, vcs_project) changeset = ChangeSet(db_project, vcs_project, now) update_entities(db_project, vcs_project, changeset) changeset.execute() return changeset.changes['obsolete_db'], removed_paths
def update_originals(db_project, now, full_scan=False): vcs_project = VCSProject(db_project, locales=[], full_scan=full_scan) with transaction.atomic(): added_paths, removed_paths, changed_paths = update_resources(db_project, vcs_project) changeset = ChangeSet(db_project, vcs_project, now) update_entities(db_project, vcs_project, changeset) changeset.execute() return added_paths, removed_paths, changed_paths
def setUp(self): # Force the checkout path to point to a test directory to make # resource file loading pass during tests. checkout_path_patch = patch.object( Project, 'checkout_path', new_callable=PropertyMock, return_value=os.path.join(TEST_CHECKOUT_PATH, 'no_resources_test') ) self.mock_checkout_path = checkout_path_patch.start() self.addCleanup(checkout_path_patch.stop) self.project = ProjectFactory.create() self.vcs_project = VCSProject(self.project)
def setUp(self): self.locale, _ = Locale.objects.get_or_create(code='fr') self.repository = RepositoryFactory() self.db_project = ProjectFactory.create(repositories=[self.repository ], ) self.resource_strings = ResourceFactory.create( project=self.db_project, path='strings.properties', ) self.resource_strings_reality = ResourceFactory.create( project=self.db_project, path='strings_reality.properties', ) # Make sure VCSConfiguration instance is initialized self.db_project.configuration_file = 'l10n.toml' self.vcs_project = VCSProject(self.db_project)
class VCSProjectTests(TestCase): def setUp(self): # Force the checkout path to point to a test directory to make # resource file loading pass during tests. checkout_path_patch = patch.object( Project, 'checkout_path', new_callable=PropertyMock, return_value=os.path.join(TEST_CHECKOUT_PATH, 'no_resources_test') ) self.mock_checkout_path = checkout_path_patch.start() self.addCleanup(checkout_path_patch.stop) self.project = ProjectFactory.create() self.vcs_project = VCSProject(self.project) def test_relative_resource_paths(self): self.vcs_project.source_directory_path = Mock(return_value='/root/') self.vcs_project.resources_for_path = Mock(return_value=[ '/root/foo.po', '/root/meh/bar.po' ]) assert_equal( list(self.vcs_project.relative_resource_paths()), ['foo.po', 'meh/bar.po'] ) def test_relative_resource_paths_pot(self): """ If a resource ends in .pot, replace the extension with .po since relative paths are used within non-source locales that do not have .pot files. """ self.vcs_project.source_directory_path = Mock(return_value='/root/') self.vcs_project.resources_for_path = Mock(return_value=[ '/root/foo.pot', '/root/meh/bar.pot' ]) assert_equal( list(self.vcs_project.relative_resource_paths()), ['foo.po', 'meh/bar.po'] ) def test_source_directory_path_no_resource(self): """ When searching for source directories, do not match directories that do not contain resource files. """ checkout_path = os.path.join(TEST_CHECKOUT_PATH, 'no_resources_test') self.mock_checkout_path.return_value = checkout_path assert_equal( self.vcs_project.source_directory_path(), os.path.join(checkout_path, 'real_resources', 'templates') ) def test_source_directory_scoring_templates(self): """ When searching for source directories, prefer directories named `templates` over all others. """ checkout_path = os.path.join(TEST_CHECKOUT_PATH, 'scoring_templates_test') self.mock_checkout_path.return_value = checkout_path assert_equal( self.vcs_project.source_directory_path(), os.path.join(checkout_path, 'templates') ) def test_source_directory_scoring_en_US(self): """ When searching for source directories, prefer directories named `en-US` over others besides `templates`. """ checkout_path = os.path.join(TEST_CHECKOUT_PATH, 'scoring_en_US_test') self.mock_checkout_path.return_value = checkout_path assert_equal( self.vcs_project.source_directory_path(), os.path.join(checkout_path, 'en-US') ) def test_source_directory_scoring_source_files(self): """ When searching for source directories, prefer directories with source-only formats over all others. """ checkout_path = os.path.join(TEST_CHECKOUT_PATH, 'scoring_source_files_test') self.mock_checkout_path.return_value = checkout_path assert_equal( self.vcs_project.source_directory_path(), os.path.join(checkout_path, 'en') # en has pot files in it ) def test_resources_parse_error(self): """ If VCSResource() raises a ParseError while loading, log an error and skip the resource. """ self.vcs_project.relative_resource_paths = Mock(return_value=['failure', 'success']) # Fail only if the path is failure so we can test the ignore. def vcs_resource_constructor(project, path, locales=None): if path == 'failure': raise ParseError('error message') else: return 'successful resource' with patch('pontoon.sync.vcs.models.VCSResource') as MockVCSResource, \ patch('pontoon.sync.vcs.models.log') as mock_log: MockVCSResource.side_effect = vcs_resource_constructor assert_equal(self.vcs_project.resources, {'success': 'successful resource'}) mock_log.error.assert_called_with(CONTAINS('failure', 'error message')) def test_resource_for_path_region_properties(self): """ If a project has a repository_url in pontoon.base.MOZILLA_REPOS, resources_for_path should ignore files named "region.properties". """ url = 'https://moz.example.com' self.project.repositories.all().delete() self.project.repositories.add(RepositoryFactory.create(url=url)) with patch('pontoon.sync.vcs.models.os', wraps=os) as mock_os, \ patch('pontoon.sync.vcs.models.MOZILLA_REPOS', [url]): mock_os.walk.return_value = [ ('/root', [], ['foo.pot', 'region.properties']) ] assert_equal( list(self.vcs_project.resources_for_path('/root')), [os.path.join('/root', 'foo.pot')] ) def test_filter_hidden_directories(self): """ We should filter out resources that are contained in the hidden paths. """ hidden_paths = ( ('/root/.hidden_folder/templates', [], ('bar.pot',)), ('/root/templates', [], ('foo.pot',)), ) with patch('pontoon.sync.vcs.models.os.walk', wraps=os, return_value=hidden_paths): assert_equal( list(self.vcs_project.resources_for_path('/root')), ['/root/templates/foo.pot'] )
def setUp(self): self.now = aware_datetime(1970, 1, 1) timezone_patch = patch("pontoon.sync.tasks.timezone") self.mock_timezone = timezone_patch.start() self.addCleanup(timezone_patch.stop) self.mock_timezone.now.return_value = self.now self.translated_locale = LocaleFactory.create(code="translated-locale") self.inactive_locale = LocaleFactory.create(code="inactive-locale") self.repository = RepositoryFactory() self.db_project = ProjectFactory.create( name="db-project", locales=[self.translated_locale], repositories=[self.repository], ) self.main_db_resource = ResourceFactory.create(project=self.db_project, path="main.lang", format="lang") self.other_db_resource = ResourceFactory.create( project=self.db_project, path="other.lang", format="lang") self.missing_db_resource = ResourceFactory.create( project=self.db_project, path="missing.lang", format="lang") self.main_db_entity = EntityFactory.create( resource=self.main_db_resource, string="Source String", key="Source String", obsolete=False, ) self.other_db_entity = EntityFactory.create( resource=self.other_db_resource, string="Other Source String", key="Other Source String", obsolete=False, ) self.main_db_translation = TranslationFactory.create( entity=self.main_db_entity, plural_form=None, locale=self.translated_locale, string="Translated String", date=aware_datetime(1970, 1, 1), approved=True, extra={"tags": []}, ) # Load paths from the fake locale directory. checkout_path_patch = patch.object( Project, "checkout_path", new_callable=PropertyMock, return_value=FAKE_CHECKOUT_PATH, ) checkout_path_patch.start() self.addCleanup(checkout_path_patch.stop) vcs_changed_files = { self.main_db_resource.path: [self.translated_locale], self.other_db_resource.path: [self.translated_locale], self.missing_db_resource.path: [self.translated_locale], } changed_files_patch = patch.object( VCSProject, "changed_files", new_callable=PropertyMock, return_value=vcs_changed_files, ) changed_files_patch.start() self.addCleanup(changed_files_patch.stop) source_repository = patch.object( Project, "source_repository", new_callable=PropertyMock, return_value=self.db_project.repositories.all()[0], ) source_repository.start() self.addCleanup(source_repository.stop) self.vcs_project = VCSProject(self.db_project) self.main_vcs_resource = self.vcs_project.resources[ self.main_db_resource.path] self.other_vcs_resource = self.vcs_project.resources[ self.other_db_resource.path] self.missing_vcs_resource = self.vcs_project.resources[ self.missing_db_resource.path] self.main_vcs_entity = self.main_vcs_resource.entities["Source String"] self.main_vcs_translation = self.main_vcs_entity.translations[ "translated-locale"] # Mock VCSResource.save() for each resource to avoid altering # the filesystem. resource_save_patch = patch.object(VCSResource, "save") resource_save_patch.start() self.addCleanup(resource_save_patch.stop) self.changeset = ChangeSet( self.db_project, self.vcs_project, aware_datetime(1970, 1, 1), self.translated_locale, )
class VCSProjectTests(TestCase): def setUp(self): # Force the checkout path to point to a test directory to make # resource file loading pass during tests. checkout_path_patch = patch.object(Project, 'checkout_path', new_callable=PropertyMock, return_value=os.path.join( TEST_CHECKOUT_PATH, 'no_resources_test')) self.mock_checkout_path = checkout_path_patch.start() self.addCleanup(checkout_path_patch.stop) self.project = ProjectFactory.create() self.vcs_project = VCSProject(self.project) def test_relative_resource_paths(self): with patch.object(VCSProject, 'source_directory_path', new_callable=PropertyMock, return_value='/root/'): self.vcs_project.resources_for_path = Mock( return_value=['/root/foo.po', '/root/meh/bar.po']) assert_equal(list(self.vcs_project.relative_resource_paths()), ['foo.po', 'meh/bar.po']) def test_relative_resource_paths_pot(self): """ If a resource ends in .pot, replace the extension with .po since relative paths are used within non-source locales that do not have .pot files. """ with patch.object(VCSProject, 'source_directory_path', new_callable=PropertyMock, return_value='/root/'): self.vcs_project.resources_for_path = Mock( return_value=['/root/foo.pot', '/root/meh/bar.pot']) assert_equal(list(self.vcs_project.relative_resource_paths()), ['foo.po', 'meh/bar.po']) def test_source_directory_path_no_resource(self): """ When searching for source directories, do not match directories that do not contain resource files. """ checkout_path = os.path.join(TEST_CHECKOUT_PATH, 'no_resources_test') self.mock_checkout_path.return_value = checkout_path assert_equal( self.vcs_project.source_directory_path, os.path.join(checkout_path, 'real_resources', 'templates')) def test_source_directory_scoring_templates(self): """ When searching for source directories, prefer directories named `templates` over all others. """ checkout_path = os.path.join(TEST_CHECKOUT_PATH, 'scoring_templates_test') self.mock_checkout_path.return_value = checkout_path assert_equal(self.vcs_project.source_directory_path, os.path.join(checkout_path, 'templates')) def test_source_directory_scoring_en_US(self): """ When searching for source directories, prefer directories named `en-US` over others besides `templates`. """ checkout_path = os.path.join(TEST_CHECKOUT_PATH, 'scoring_en_US_test') self.mock_checkout_path.return_value = checkout_path assert_equal(self.vcs_project.source_directory_path, os.path.join(checkout_path, 'en-US')) def test_source_directory_scoring_source_files(self): """ When searching for source directories, prefer directories with source-only formats over all others. """ checkout_path = os.path.join(TEST_CHECKOUT_PATH, 'scoring_source_files_test') self.mock_checkout_path.return_value = checkout_path assert_equal( self.vcs_project.source_directory_path, os.path.join(checkout_path, 'en') # en has pot files in it ) def test_resources_parse_error(self): """ If VCSResource() raises a ParseError while loading, log an error and skip the resource. """ self.vcs_project.relative_resource_paths = Mock( return_value=['failure', 'success']) # Fail only if the path is failure so we can test the ignore. def vcs_resource_constructor(project, path, locales=None): if path == 'failure': raise ParseError('error message') else: return 'successful resource' changed_vcs_resources = {'success': [], 'failure': []} with patch('pontoon.sync.vcs.models.VCSResource') as MockVCSResource, \ patch('pontoon.sync.vcs.models.log') as mock_log, \ patch.object( VCSProject, 'changed_files', new_callable=PropertyMock, return_value=changed_vcs_resources ): MockVCSResource.side_effect = vcs_resource_constructor assert_equal(self.vcs_project.resources, {'success': 'successful resource'}) mock_log.error.assert_called_with( CONTAINS('failure', 'error message')) def test_resource_for_path_region_properties(self): """ If a project has a repository_url in pontoon.base.MOZILLA_REPOS, resources_for_path should ignore files named "region.properties". """ url = 'https://moz.example.com' self.project.repositories.all().delete() self.project.repositories.add(RepositoryFactory.create(url=url)) with patch('pontoon.sync.vcs.models.scandir', wraps=scandir) as mock_scandir, \ patch( 'pontoon.sync.vcs.models.MOZILLA_REPOS', [url] ): mock_scandir.walk.return_value = [ ('/root', [], ['foo.pot', 'region.properties']) ] assert_equal(list(self.vcs_project.resources_for_path('/root')), [os.path.join('/root', 'foo.pot')]) def test_filter_hidden_directories(self): """ We should filter out resources that are contained in the hidden paths. """ hidden_paths = ( ('/root/.hidden_folder/templates', [], ('bar.pot', )), ('/root/templates', [], ('foo.pot', )), ) with patch('pontoon.sync.vcs.models.scandir.walk', wraps=scandir, return_value=hidden_paths): assert_equal(list(self.vcs_project.resources_for_path('/root')), ['/root/templates/foo.pot'])
def sync_translations( self, project_pk, project_sync_log_pk, now, added_paths=None, removed_paths=None, changed_paths=None, new_entities=None, locale=None, no_pull=False, no_commit=False, full_scan=False, ): db_project = get_or_fail( Project, pk=project_pk, message="Could not sync project with pk={0}, not found.".format(project_pk), ) repos = db_project.translation_repositories() repo_pk = repos[0].pk repo = get_or_fail( Repository, pk=repo_pk, message="Could not sync repo with pk={0}, not found.".format(repo_pk), ) project_sync_log = get_or_fail( ProjectSyncLog, pk=project_sync_log_pk, message=( "Could not sync project {0}, log with pk={1} not found.".format( db_project.slug, project_sync_log_pk ) ), ) log.info("Syncing translations for project: {}".format(db_project.slug)) repo_sync_log = RepositorySyncLog.objects.create( project_sync_log=project_sync_log, repository=repo, start_time=timezone.now() ) if locale: locales = db_project.locales.filter(pk=locale.pk) else: locales = db_project.locales.all() if not locales: log.info( "Skipping syncing translations for project {0}, no locales to sync " "found within.".format(db_project.slug) ) repo_sync_log.end() return # If project repositories have API access, we can retrieve latest commit hashes and detect # changed locales before the expensive VCS pull/clone operations. When performing full scan, # we still need to sync all locales. if not full_scan: locales = get_changed_locales(db_project, locales, now) readonly_locales = db_project.locales.filter(project_locale__readonly=True) added_and_changed_resources = db_project.resources.filter( path__in=list(added_paths or []) + list(changed_paths or []) ).distinct() # We should also sync files for which source file change - but only for read-only locales. # See bug 1372151 for more details. if added_and_changed_resources: changed_locales_pks = [l.pk for l in locales] readonly_locales_pks = [l.pk for l in readonly_locales] locales = db_project.locales.filter( pk__in=changed_locales_pks + readonly_locales_pks ) # Pull VCS changes in case we're on a different worker than the one # sync started on. if not no_pull: log.info("Pulling changes for project {0} started.".format(db_project.slug)) repos_changed, repo_locales = pull_changes(db_project, locales) repos = repos.filter(pk__in=repo_locales.keys()) log.info("Pulling changes for project {0} complete.".format(db_project.slug)) # If none of the repos has changed since the last sync and there are # no Pontoon-side changes for this project, quit early. if ( not full_scan and not db_project.needs_sync and not repos_changed and not (added_paths or removed_paths or changed_paths) ): log.info("Skipping project {0}, no changes detected.".format(db_project.slug)) repo_sync_log.end() return vcs_project = VCSProject( db_project, now, locales=locales, repo_locales=repo_locales, added_paths=added_paths, changed_paths=changed_paths, full_scan=full_scan, ) synced_locales = set() failed_locales = set() # Store newly added locales and locales with newly added resources new_locales = [] for locale in locales: try: with transaction.atomic(): # Sets VCSProject.synced_locales, needed to skip early if not vcs_project.synced_locales: vcs_project.resources # Skip all locales if none of the them has anything to sync if len(vcs_project.synced_locales) == 0: break # Skip locales that have nothing to sync if ( vcs_project.synced_locales and locale not in vcs_project.synced_locales ): continue changeset = ChangeSet(db_project, vcs_project, now, locale) update_translations(db_project, vcs_project, locale, changeset) changeset.execute() created = update_translated_resources(db_project, vcs_project, locale) if created: new_locales.append(locale.pk) update_locale_project_locale_stats(locale, db_project) # Clear out the "has_changed" markers now that we've finished # syncing. ( ChangedEntityLocale.objects.filter( entity__resource__project=db_project, locale=locale, when__lte=now, ).delete() ) # Perform the commit last so that, if it succeeds, there is # nothing after it to fail. if ( not no_commit and locale in changeset.locales_to_commit and locale not in readonly_locales ): commit_changes(db_project, vcs_project, changeset, locale) log.info( "Synced locale {locale} for project {project}.".format( locale=locale.code, project=db_project.slug, ) ) synced_locales.add(locale.code) except CommitToRepositoryException as err: # Transaction aborted, log and move on to the next locale. log.warning( "Failed to sync locale {locale} for project {project} due to " "commit error: {error}".format( locale=locale.code, project=db_project.slug, error=err, ) ) failed_locales.add(locale.code) # If sources have changed, update stats for all locales. if added_paths or removed_paths or changed_paths: for locale in db_project.locales.all(): # Already synced. if locale.code in synced_locales: continue # We have files: update all translated resources. if locale in locales: created = update_translated_resources(db_project, vcs_project, locale) if created: new_locales.append[locale.pk] # We don't have files: we can still update asymmetric translated resources. else: update_translated_resources_no_files( db_project, locale, added_and_changed_resources, ) update_locale_project_locale_stats(locale, db_project) synced_locales.add(locale.code) log.info( "Synced source changes for locale {locale} for project {project}.".format( locale=locale.code, project=db_project.slug, ) ) db_project.aggregate_stats() if synced_locales: log.info( "Synced translations for project {0} in locales {1}.".format( db_project.slug, ",".join(synced_locales) ) ) elif failed_locales: log.info( "Failed to sync translations for project {0} due to commit error.".format( db_project.slug ) ) else: log.info( "Skipping syncing translations for project {0}, none of the locales " "has anything to sync.".format(db_project.slug) ) for r in repos: r.set_last_synced_revisions( locales=repo_locales[r.pk].exclude(code__in=failed_locales) ) repo_sync_log.end() if db_project.pretranslation_enabled: # Pretranslate all entities for newly added locales # and locales with newly added resources if len(new_locales): pretranslate(db_project, locales=new_locales) locales = db_project.locales.exclude(pk__in=new_locales).values_list( "pk", flat=True ) # Pretranslate newly added entities for all locales if new_entities and locales: new_entities = list(set(new_entities)) pretranslate(db_project, locales=locales, entities=new_entities)
def sync_project_repo(self, project_pk, repo_pk, project_sync_log_pk, now, obsolete_vcs_entities=None, obsolete_vcs_resources=None, new_paths=None, locale=None, no_pull=False, no_commit=False, full_scan=False): db_project = get_or_fail( Project, pk=project_pk, message='Could not sync project with pk={0}, not found.'.format( project_pk)) repo = get_or_fail( Repository, pk=repo_pk, message='Could not sync repo with pk={0}, not found.'.format(repo_pk)) project_sync_log = get_or_fail( ProjectSyncLog, pk=project_sync_log_pk, message=( 'Could not sync project {0}, log with pk={1} not found.'.format( db_project.slug, project_sync_log_pk))) log.info('Syncing repo: {}'.format(repo.url)) repo_sync_log = RepositorySyncLog.objects.create( project_sync_log=project_sync_log, repository=repo, start_time=timezone.now()) # Pull VCS changes in case we're on a different worker than the one # sync started on. if not no_pull: pull_changes(db_project) if locale: locales = [locale] else: locales = repo.locales # Cannot skip earlier - repo.locales is only available after pull_changes() if not locales: log.debug( 'Skipping repo `{0}` for project {1}, no locales to sync found within.' .format(repo.url, db_project.slug)) repo_sync_log.end() return vcs_project = VCSProject( db_project, locales=locales, obsolete_entities_paths=Resource.objects.obsolete_entities_paths( obsolete_vcs_entities), new_paths=new_paths, full_scan=full_scan) for locale in locales: try: with transaction.atomic(): # Skip locales that have nothing to sync if vcs_project.synced_locales and locale not in vcs_project.synced_locales: if obsolete_vcs_entities or obsolete_vcs_resources: update_locale_project_locale_stats(locale, db_project) continue changeset = ChangeSet(db_project, vcs_project, now, obsolete_vcs_entities, obsolete_vcs_resources) update_translations(db_project, vcs_project, locale, changeset) changeset.execute() update_translated_resources(db_project, vcs_project, changeset, locale) # Skip if none of the locales has anything to sync # VCSProject.synced_locales is set on a first call to # VCSProject.resources, which is set in # pontoon.sync.core.update_translated_resources() if len(vcs_project.synced_locales) == 0: if obsolete_vcs_entities or obsolete_vcs_resources: for l in locales: update_locale_project_locale_stats(l, db_project) db_project.aggregate_stats() log.info( 'Skipping repo `{0}` for project {1}, none of the locales has anything to sync.' .format(repo.url, db_project.slug)) repo_sync_log.end() return update_locale_project_locale_stats(locale, db_project) # Clear out the "has_changed" markers now that we've finished # syncing. (ChangedEntityLocale.objects.filter( entity__resource__project=db_project, locale=locale, when__lte=now).delete()) # Clean up any duplicate approvals at the end of sync right # before we commit the transaction to avoid race conditions. with connection.cursor() as cursor: cursor.execute( """ UPDATE base_translation AS b SET approved = FALSE, approved_date = NULL WHERE id IN (SELECT trans.id FROM base_translation AS trans LEFT JOIN base_entity AS ent ON ent.id = trans.entity_id LEFT JOIN base_resource AS res ON res.id = ent.resource_id WHERE locale_id = %(locale_id)s AND res.project_id = %(project_id)s) AND approved_date != (SELECT max(approved_date) FROM base_translation WHERE entity_id = b.entity_id AND locale_id = b.locale_id AND (plural_form = b.plural_form OR plural_form IS NULL)); """, { 'locale_id': locale.id, 'project_id': db_project.id }) # Perform the commit last so that, if it succeeds, there is # nothing after it to fail. if not no_commit and locale in changeset.locales_to_commit: commit_changes(db_project, vcs_project, changeset, locale) except CommitToRepositoryException as err: # Transaction aborted, log and move on to the next locale. log.warning( 'Failed to sync locale {locale} for project {project} due to ' 'commit error: {error}'.format( locale=locale.code, project=db_project.slug, error=err, )) with transaction.atomic(): db_project.aggregate_stats() log.info('Synced translations for project {0} in locales {1}.'.format( db_project.slug, ','.join(locale.code for locale in vcs_project.synced_locales))) repo_sync_log.end()
def sync_translations(self, project_pk, project_sync_log_pk, now, project_changes=None, obsolete_vcs_resources=None, new_paths=None, locale=None, no_pull=False, no_commit=False, full_scan=False): db_project = get_or_fail( Project, pk=project_pk, message='Could not sync project with pk={0}, not found.'.format( project_pk)) repos = db_project.translation_repositories() repo_pk = repos[0].pk repo = get_or_fail( Repository, pk=repo_pk, message='Could not sync repo with pk={0}, not found.'.format(repo_pk)) project_sync_log = get_or_fail( ProjectSyncLog, pk=project_sync_log_pk, message=( 'Could not sync project {0}, log with pk={1} not found.'.format( db_project.slug, project_sync_log_pk))) log.info('Syncing translations for project: {}'.format(db_project.slug)) repo_sync_log = RepositorySyncLog.objects.create( project_sync_log=project_sync_log, repository=repo, start_time=timezone.now()) if locale: locales = db_project.locales.filter(pk=locale.pk) else: locales = db_project.locales.all() if not locales: log.info( 'Skipping syncing translations for project {0}, no locales to sync found within.' .format(db_project.slug)) repo_sync_log.end() return # If project repositories have API access, we can retrieve latest commit hashes and detect # changed locales before the expensive VCS pull/clone operations. When performing full scan, # we still need to sync all locales. if not full_scan: locales = get_changed_locales(db_project, locales, now) # Pull VCS changes in case we're on a different worker than the one # sync started on. if not no_pull: log.info('Pulling changes for project {0} started.'.format( db_project.slug)) repos_changed, repo_locales = pull_changes(db_project, locales) repos = repos.filter(pk__in=repo_locales.keys()) log.info('Pulling changes for project {0} complete.'.format( db_project.slug)) changed_resources = [] obsolete_vcs_entities = [] if project_changes: updated_entity_pks = [] for locale_code, db_entity, vcs_entity in project_changes['update_db']: updated_entity_pks.append(db_entity.pk) obsolete_entity_pks = project_changes['obsolete_db'] changed_resources = db_project.resources.filter( Q(entities__date_created=now) | Q(entities__pk__in=updated_entity_pks + obsolete_entity_pks)).distinct() obsolete_vcs_entities = project_changes['obsolete_db'] # If none of the repos has changed since the last sync and there are # no Pontoon-side changes for this project, quit early. if (not full_scan and not db_project.needs_sync and not repos_changed and not (changed_resources or obsolete_vcs_resources)): log.info('Skipping project {0}, no changes detected.'.format( db_project.slug)) repo_sync_log.end() return obsolete_entities_paths = ( Resource.objects.obsolete_entities_paths(obsolete_vcs_entities) if obsolete_vcs_entities else None) vcs_project = VCSProject(db_project, now, locales=locales, repo_locales=repo_locales, obsolete_entities_paths=obsolete_entities_paths, new_paths=new_paths, full_scan=full_scan) synced_locales = set() failed_locales = set() for locale in locales: try: with transaction.atomic(): # Sets VCSProject.synced_locales, needed to skip early if not vcs_project.synced_locales: vcs_project.resources # Skip all locales if none of the them has anything to sync if len(vcs_project.synced_locales) == 0: break # Skip locales that have nothing to sync if vcs_project.synced_locales and locale not in vcs_project.synced_locales: continue changeset = ChangeSet(db_project, vcs_project, now, locale) update_translations(db_project, vcs_project, locale, changeset) changeset.execute() update_translated_resources(db_project, vcs_project, locale) update_locale_project_locale_stats(locale, db_project) # Clear out the "has_changed" markers now that we've finished # syncing. (ChangedEntityLocale.objects.filter( entity__resource__project=db_project, locale=locale, when__lte=now).delete()) # Clean up any duplicate approvals at the end of sync right # before we commit the transaction to avoid race conditions. with connection.cursor() as cursor: cursor.execute( """ UPDATE base_translation AS b SET approved = FALSE, approved_date = NULL WHERE id IN (SELECT trans.id FROM base_translation AS trans LEFT JOIN base_entity AS ent ON ent.id = trans.entity_id LEFT JOIN base_resource AS res ON res.id = ent.resource_id WHERE locale_id = %(locale_id)s AND res.project_id = %(project_id)s) AND approved_date != (SELECT max(approved_date) FROM base_translation WHERE entity_id = b.entity_id AND locale_id = b.locale_id AND (plural_form = b.plural_form OR plural_form IS NULL)); """, { 'locale_id': locale.id, 'project_id': db_project.id }) # Perform the commit last so that, if it succeeds, there is # nothing after it to fail. if not no_commit and locale in changeset.locales_to_commit: commit_changes(db_project, vcs_project, changeset, locale) log.info( 'Synced locale {locale} for project {project}.'.format( locale=locale.code, project=db_project.slug, )) synced_locales.add(locale.code) except CommitToRepositoryException as err: # Transaction aborted, log and move on to the next locale. log.warning( 'Failed to sync locale {locale} for project {project} due to ' 'commit error: {error}'.format( locale=locale.code, project=db_project.slug, error=err, )) failed_locales.add(locale.code) # If sources have changed, update stats for all locales. if changed_resources or obsolete_vcs_resources: for locale in db_project.locales.all(): # Already synced. if locale.code in synced_locales: continue # We have files: update all translated resources. if locale in locales: update_translated_resources(db_project, vcs_project, locale) # We don't have files: we can still update asymmetric translated resources. else: update_translated_resources_no_files(db_project, locale, changed_resources) update_locale_project_locale_stats(locale, db_project) synced_locales.add(locale.code) log.info( 'Synced source changes for locale {locale} for project {project}.' .format( locale=locale.code, project=db_project.slug, )) db_project.aggregate_stats() if synced_locales: log.info('Synced translations for project {0} in locales {1}.'.format( db_project.slug, ','.join(synced_locales))) elif failed_locales: log.info( 'Failed to sync translations for project {0} due to commit error.'. format(db_project.slug)) else: log.info( 'Skipping syncing translations for project {0}, none of the locales ' 'has anything to sync.'.format(db_project.slug)) for r in repos: r.set_last_synced_revisions(locales=repo_locales[r.pk].exclude( code__in=failed_locales)) repo_sync_log.end()
def sync_project_repo(self, project_pk, repo_pk, project_sync_log_pk, now, obsolete_vcs=None, no_pull=False, no_commit=False): db_project = get_or_fail( Project, pk=project_pk, message='Could not sync project with pk={0}, not found.'.format( project_pk)) repo = get_or_fail( Repository, pk=repo_pk, message='Could not sync repo with pk={0}, not found.'.format( project_pk)) project_sync_log = get_or_fail( ProjectSyncLog, pk=project_sync_log_pk, message=( 'Could not sync project {0}, log with pk={1} not found.'.format( db_project.slug, project_sync_log_pk))) repo_sync_log = RepositorySyncLog.objects.create( project_sync_log=project_sync_log, repository=repo, start_time=timezone.now()) # Pull VCS changes in case we're on a different worker than the one # sync started on. if not no_pull: pull_changes(db_project) if len(repo.locales) < 1: log.warning( 'Could not sync repo `{0}`, no locales found within.'.format( repo.url)) repo_sync_log.end_time = timezone.now() repo_sync_log.save(update_fields=['end_time']) return vcs_project = VCSProject(db_project, locales=repo.locales) for locale in repo.locales: try: with transaction.atomic(): changeset = ChangeSet(db_project, vcs_project, now, obsolete_vcs) update_translations(db_project, vcs_project, locale, changeset) changeset.execute() update_translated_resources(db_project, vcs_project, changeset, locale) # Clear out the "has_changed" markers now that we've finished # syncing. (ChangedEntityLocale.objects.filter( entity__resource__project=db_project, locale=locale, when__lte=now).delete()) db_project.has_changed = False db_project.save(update_fields=['has_changed']) # Clean up any duplicate approvals at the end of sync right # before we commit the transaction to avoid race conditions. with connection.cursor() as cursor: cursor.execute( """ UPDATE base_translation AS b SET approved = FALSE, approved_date = NULL WHERE id IN (SELECT trans.id FROM base_translation AS trans LEFT JOIN base_entity AS ent ON ent.id = trans.entity_id LEFT JOIN base_resource AS res ON res.id = ent.resource_id WHERE locale_id = %(locale_id)s AND res.project_id = %(project_id)s) AND approved_date != (SELECT max(approved_date) FROM base_translation WHERE entity_id = b.entity_id AND locale_id = b.locale_id AND (plural_form = b.plural_form OR plural_form IS NULL)); """, { 'locale_id': locale.id, 'project_id': db_project.id }) # Perform the commit last so that, if it succeeds, there is # nothing after it to fail. if not no_commit and locale in changeset.locales_to_commit: commit_changes(db_project, vcs_project, changeset, locale) except CommitToRepositoryException as err: # Transaction aborted, log and move on to the next locale. log.warning( 'Failed to sync locale {locale} for project {project} due to ' 'commit error: {error}'.format( locale=locale.code, project=db_project.slug, error=err, )) repo_sync_log.end_time = timezone.now() repo_sync_log.save() log.info('Synced translations for project {0} in locales {1}.'.format( db_project.slug, ','.join(locale.code for locale in repo.locales)))
def get_download_content(slug, code, part): """ Get content of the file to be downloaded. :arg str slug: Project slug. :arg str code: Locale code. :arg str part: Resource path or Subpage name. """ # Avoid circular import; someday we should refactor to avoid. from pontoon.sync import formats from pontoon.sync.utils import source_to_locale_path from pontoon.sync.vcs.models import VCSProject from pontoon.base.models import Entity, Locale, Project, Resource project = get_object_or_404(Project, slug=slug) locale = get_object_or_404(Locale, code=code) vcs_project = VCSProject(project, locales=[locale]) # Download a ZIP of all files if project has > 1 and < 10 resources resources = Resource.objects.filter(project=project, translatedresources__locale=locale) isZipable = 1 < len(resources) < 10 if isZipable: s = io.BytesIO() zf = zipfile.ZipFile(s, "w") # Download a single file if project has 1 or >= 10 resources else: relative_path = _get_relative_path_from_part(slug, part) resources = [ get_object_or_404(Resource, project__slug=slug, path=relative_path) ] locale_prefixes = project.repositories if not project.configuration_file: locale_prefixes = locale_prefixes.filter( permalink_prefix__contains="{locale_code}") locale_prefixes = locale_prefixes.values_list("permalink_prefix", flat=True).distinct() source_prefixes = project.repositories.values_list("permalink_prefix", flat=True).distinct() for resource in resources: # Get locale file dirnames = set([locale.code, locale.code.replace("-", "_")]) locale_path = _download_file(locale_prefixes, dirnames, vcs_project, resource.path) if not locale_path and not resource.is_asymmetric: return None, None # Get source file if needed source_path = None if resource.is_asymmetric: dirnames = VCSProject.SOURCE_DIR_NAMES source_path = _download_file(source_prefixes, dirnames, vcs_project, resource.path) if not source_path: return None, None # If locale file doesn't exist, create it if not locale_path: extension = os.path.splitext(resource.path)[1] with tempfile.NamedTemporaryFile( prefix="strings" if extension == ".xml" else "", suffix=extension, delete=False, ) as temp: temp.flush() locale_path = temp.name # Update file from database resource_file = formats.parse(locale_path, source_path) entities_dict = {} entities_qs = Entity.objects.filter( changedentitylocale__locale=locale, resource__project=project, resource__path=resource.path, obsolete=False, ) for e in entities_qs: entities_dict[e.key] = e.translation_set.filter(approved=True, locale=locale) for vcs_translation in resource_file.translations: key = vcs_translation.key if key in entities_dict: entity = entities_dict[key] vcs_translation.update_from_db(entity) resource_file.save(locale) if not locale_path: return None, None if isZipable: zf.write(locale_path, source_to_locale_path(resource.path)) else: with codecs.open(locale_path, "r", "utf-8") as f: content = f.read() filename = os.path.basename(source_to_locale_path(resource.path)) # Remove temporary files os.remove(locale_path) if source_path: os.remove(source_path) if isZipable: zf.close() content = s.getvalue() filename = project.slug + ".zip" return content, filename
def setUp(self): self.now = aware_datetime(1970, 1, 1) timezone_patch = patch('pontoon.sync.tasks.timezone') self.mock_timezone = timezone_patch.start() self.addCleanup(timezone_patch.stop) self.mock_timezone.now.return_value = self.now self.translated_locale = LocaleFactory.create(code='translated-locale') self.inactive_locale = LocaleFactory.create(code='inactive-locale') self.repository = RepositoryFactory() self.db_project = ProjectFactory.create( name='db-project', locales=[self.translated_locale], repositories=[self.repository]) self.main_db_resource = ResourceFactory.create(project=self.db_project, path='main.lang', format='lang') self.other_db_resource = ResourceFactory.create( project=self.db_project, path='other.lang', format='lang') self.missing_db_resource = ResourceFactory.create( project=self.db_project, path='missing.lang', format='lang') self.main_db_entity = EntityFactory.create( resource=self.main_db_resource, string='Source String', key='Source String', obsolete=False) self.other_db_entity = EntityFactory.create( resource=self.other_db_resource, string='Other Source String', key='Other Source String', obsolete=False) self.main_db_translation = TranslationFactory.create( entity=self.main_db_entity, plural_form=None, locale=self.translated_locale, string='Translated String', date=aware_datetime(1970, 1, 1), approved=True, extra={'tags': []}) # Load paths from the fake locale directory. checkout_path_patch = patch.object(Project, 'checkout_path', new_callable=PropertyMock, return_value=FAKE_CHECKOUT_PATH) checkout_path_patch.start() self.addCleanup(checkout_path_patch.stop) self.vcs_project = VCSProject(self.db_project) self.main_vcs_resource = self.vcs_project.resources[ self.main_db_resource.path] self.other_vcs_resource = self.vcs_project.resources[ self.other_db_resource.path] self.missing_vcs_resource = self.vcs_project.resources[ self.missing_db_resource.path] self.main_vcs_entity = self.main_vcs_resource.entities['Source String'] self.main_vcs_translation = self.main_vcs_entity.translations[ 'translated-locale'] # Mock VCSResource.save() for each resource to avoid altering # the filesystem. resource_save_patch = patch.object(VCSResource, 'save') resource_save_patch.start() self.addCleanup(resource_save_patch.stop) self.changeset = ChangeSet(self.db_project, self.vcs_project, aware_datetime(1970, 1, 1))
def handle_upload_content(slug, code, part, f, user): """ Update translations in the database from uploaded file. :arg str slug: Project slug. :arg str code: Locale code. :arg str part: Resource path or Subpage name. :arg UploadedFile f: UploadedFile instance. :arg User user: User uploading the file. """ # Avoid circular import; someday we should refactor to avoid. from pontoon.sync import formats from pontoon.sync.changeset import ChangeSet from pontoon.sync.vcs.models import VCSProject from pontoon.base.models import ( ChangedEntityLocale, Entity, Locale, Project, Resource, TranslatedResource, Translation, ) relative_path = _get_relative_path_from_part(slug, part) project = get_object_or_404(Project, slug=slug) locale = get_object_or_404(Locale, code=code) resource = get_object_or_404(Resource, project__slug=slug, path=relative_path) # Store uploaded file to a temporary file and parse it extension = os.path.splitext(f.name)[1] with tempfile.NamedTemporaryFile(suffix=extension) as temp: for chunk in f.chunks(): temp.write(chunk) temp.flush() resource_file = formats.parse(temp.name) # Update database objects from file changeset = ChangeSet(project, VCSProject(project, locales=[locale]), timezone.now()) entities_qs = Entity.objects.filter( resource__project=project, resource__path=relative_path, obsolete=False).prefetch_related( Prefetch('translation_set', queryset=Translation.objects.filter(locale=locale), to_attr='db_translations')).prefetch_related( Prefetch( 'translation_set', queryset=Translation.objects.filter( locale=locale, approved_date__lte=timezone.now()), to_attr='db_translations_approved_before_sync')) entities_dict = {entity.key: entity for entity in entities_qs} for vcs_translation in resource_file.translations: key = vcs_translation.key if key in entities_dict: entity = entities_dict[key] changeset.update_entity_translations_from_vcs( entity, locale.code, vcs_translation, user, entity.db_translations, entity.db_translations_approved_before_sync) changeset.bulk_create_translations() changeset.bulk_update_translations() changeset.bulk_create_translaton_memory_entries() TranslatedResource.objects.get(resource=resource, locale=locale).calculate_stats() # Mark translations as changed changed_entities = {} existing = ChangedEntityLocale.objects.values_list('entity', 'locale').distinct() for t in changeset.changed_translations: key = (t.entity.pk, t.locale.pk) # Remove duplicate changes to prevent unique constraint violation if key not in existing: changed_entities[key] = ChangedEntityLocale(entity=t.entity, locale=t.locale) ChangedEntityLocale.objects.bulk_create(changed_entities.values()) # Update latest translation if changeset.translations_to_create: changeset.translations_to_create[-1].update_latest_translation()
def handle_upload_content(slug, code, part, f, user): """ Update translations in the database from uploaded file. :arg str slug: Project slug. :arg str code: Locale code. :arg str part: Resource path or Subpage name. :arg UploadedFile f: UploadedFile instance. :arg User user: User uploading the file. """ # Avoid circular import; someday we should refactor to avoid. from pontoon.sync import formats from pontoon.sync.changeset import ChangeSet from pontoon.sync.vcs.models import VCSProject from pontoon.base.models import ( ChangedEntityLocale, Entity, Locale, Project, Resource, TranslatedResource, Translation, ) relative_path = _get_relative_path_from_part(slug, part) project = get_object_or_404(Project, slug=slug) locale = get_object_or_404(Locale, code=code) resource = get_object_or_404(Resource, project__slug=slug, path=relative_path) # Store uploaded file to a temporary file and parse it extension = os.path.splitext(f.name)[1] with tempfile.NamedTemporaryFile( prefix="strings" if extension == ".xml" else "", suffix=extension, ) as temp: for chunk in f.chunks(): temp.write(chunk) temp.flush() resource_file = formats.parse(temp.name) # Update database objects from file changeset = ChangeSet(project, VCSProject(project, locales=[locale]), timezone.now()) entities_qs = (Entity.objects.filter( resource__project=project, resource__path=relative_path, obsolete=False).prefetch_related( Prefetch( "translation_set", queryset=Translation.objects.filter(locale=locale), to_attr="db_translations", )).prefetch_related( Prefetch( "translation_set", queryset=Translation.objects.filter( locale=locale, approved_date__lte=timezone.now()), to_attr="db_translations_approved_before_sync", ))) entities_dict = {entity.key: entity for entity in entities_qs} for vcs_translation in resource_file.translations: key = vcs_translation.key if key in entities_dict: entity = entities_dict[key] changeset.update_entity_translations_from_vcs( entity, locale.code, vcs_translation, user, entity.db_translations, entity.db_translations_approved_before_sync, ) changeset.bulk_create_translations() changeset.bulk_update_translations() changeset.bulk_log_actions() if changeset.changed_translations: # Update 'active' status of all changed translations and their siblings, # i.e. translations of the same entity to the same locale. changed_pks = {t.pk for t in changeset.changed_translations} (Entity.objects.filter( translation__pk__in=changed_pks).reset_active_translations( locale=locale)) # Run checks and create TM entries for translations that pass them valid_translations = changeset.bulk_check_translations() changeset.bulk_create_translation_memory_entries(valid_translations) # Remove any TM entries of translations that got rejected changeset.bulk_remove_translation_memory_entries() TranslatedResource.objects.get(resource=resource, locale=locale).calculate_stats() # Mark translations as changed changed_entities = {} existing = ChangedEntityLocale.objects.values_list("entity", "locale").distinct() for t in changeset.changed_translations: key = (t.entity.pk, t.locale.pk) # Remove duplicate changes to prevent unique constraint violation if key not in existing: changed_entities[key] = ChangedEntityLocale(entity=t.entity, locale=t.locale) ChangedEntityLocale.objects.bulk_create(changed_entities.values()) # Update latest translation if changeset.translations_to_create: changeset.translations_to_create[-1].update_latest_translation()