def test_error(self): """If commit_changes returns an error object, log it.""" self.repository.commit.return_value = {'message': 'Whoops!'} self.db_project.repository_for_path = Mock(return_value=self.repository) with patch('pontoon.sync.core.log') as mock_log: commit_changes(self.db_project, self.vcs_project, self.changeset) mock_log.info.assert_called_with( CONTAINS('db-project', 'failed', 'Whoops!') )
def test_raised_valueerror(self): """ If db_project.repository_for_path raises a ValueError, log it. """ self.db_project.repository_for_path = Mock(side_effect=ValueError('Whoops!')) with patch('pontoon.sync.core.log') as mock_log: commit_changes(self.db_project, self.vcs_project, self.changeset) mock_log.info.assert_called_with( CONTAINS('db-project', 'failed', 'Whoops!') )
def test_raised_committorepositoryexception(self): """ If repo.commit raises a CommitToRepositoryException, log it. """ self.repository.commit.side_effect = CommitToRepositoryException('Whoops!') self.db_project.repository_for_path = Mock(return_value=self.repository) with patch('pontoon.sync.core.log') as mock_log: commit_changes(self.db_project, self.vcs_project, self.changeset) mock_log.info.assert_called_with( CONTAINS('db-project', 'failed', 'Whoops!') )
def test_basic(self): user = UserFactory.create() self.changeset.commit_authors_per_locale = { self.translated_locale.code: [user] } self.db_project.repository_for_path = Mock(return_value=self.repository) commit_changes(self.db_project, self.vcs_project, self.changeset) self.repository.commit.assert_called_with( CONTAINS(user.display_name), user, os.path.join(FAKE_CHECKOUT_PATH, self.translated_locale.code) )
def test_multiple_authors(self): """ Tests if multiple authors are passed to commit message. """ first_author, second_author = UserFactory.create_batch(2) self.changeset.commit_authors_per_locale = { self.translated_locale.code: [first_author, second_author] } self.db_project.repository_for_path = Mock(return_value=self.repository) commit_changes(self.db_project, self.vcs_project, self.changeset) self.repository.commit.assert_called_with( CONTAINS(first_author.display_name, second_author.display_name), first_author, os.path.join(FAKE_CHECKOUT_PATH, self.translated_locale.code) )
def test_author_with_multiple_contributions(self): """ Tests if author with multiple contributions occurs once in commit message. """ author = UserFactory.create() self.changeset.commit_authors_per_locale = {self.translated_locale.code: [author, author]} self.db_project.repository_for_path = Mock(return_value=self.repository) commit_changes(self.db_project, self.vcs_project, self.changeset, self.translated_locale) self.repository.commit.assert_called_with( CONTAINS(author.display_name_and_email), author, os.path.join(FAKE_CHECKOUT_PATH, self.translated_locale.code), ) commit_message = self.repository.commit.mock_calls[0][1][0] assert_equal(commit_message.count(author.display_name_and_email), 1)
def test_no_authors(self): """ If no authors are found in the changeset, default to a fake "Mozilla Pontoon" user. """ self.changeset.commit_authors_per_locale = {self.translated_locale.code: []} self.db_project.repository_for_path = Mock(return_value=self.repository) commit_changes(self.db_project, self.vcs_project, self.changeset, self.translated_locale) self.repository.commit.assert_called_with( NOT(CONTAINS("Authors:")), # Don't list authors in commit ANY, os.path.join(FAKE_CHECKOUT_PATH, self.translated_locale.code), ) user = self.mock_repo_commit.call_args[0][1] assert_equal(user.first_name, "Mozilla Pontoon") assert_equal(user.email, "*****@*****.**")
def sync_project_repo(self, project_pk, repo_pk, project_sync_log_pk, now, obsolete_vcs_entities=None, obsolete_vcs_resources=None, locale=None, no_pull=False, no_commit=False, full_scan=False): db_project = get_or_fail(Project, pk=project_pk, message='Could not sync project with pk={0}, not found.'.format(project_pk)) repo = get_or_fail(Repository, pk=repo_pk, message='Could not sync repo with pk={0}, not found.'.format(repo_pk)) project_sync_log = get_or_fail(ProjectSyncLog, pk=project_sync_log_pk, message=('Could not sync project {0}, log with pk={1} not found.' .format(db_project.slug, project_sync_log_pk))) log.info('Syncing repo: {}'.format(repo.url)) repo_sync_log = RepositorySyncLog.objects.create( project_sync_log=project_sync_log, repository=repo, start_time=timezone.now() ) # Pull VCS changes in case we're on a different worker than the one # sync started on. if not no_pull: pull_changes(db_project) if locale: locales = [locale] else: locales = repo.locales vcs_project = VCSProject( db_project, locales=locales, obsolete_entities_paths=Resource.objects.obsolete_entities_paths(obsolete_vcs_entities), full_scan=full_scan ) for locale in locales: try: with transaction.atomic(): # Skip locales that have nothing to sync if vcs_project.synced_locales and locale not in vcs_project.synced_locales: continue changeset = ChangeSet(db_project, vcs_project, now, obsolete_vcs_entities, obsolete_vcs_resources) update_translations(db_project, vcs_project, locale, changeset) changeset.execute() update_translated_resources(db_project, vcs_project, changeset, locale) # Skip if none of the locales has anything to sync # VCSProject.synced_locales is set on a first call to # VCSProject.resources, which is set in # pontoon.sync.core.update_translated_resources() if len(vcs_project.synced_locales) == 0: log.info('Skipping repo `{0}` for project {1}, none of the locales has anything to sync.' .format(repo.url, db_project.slug)) end_repo_sync(repo, repo_sync_log) return locale.aggregate_stats() locale.project_locale.get(project=db_project).aggregate_stats() # Clear out the "has_changed" markers now that we've finished # syncing. (ChangedEntityLocale.objects .filter(entity__resource__project=db_project, locale=locale, when__lte=now) .delete()) db_project.has_changed = False db_project.save(update_fields=['has_changed']) # Clean up any duplicate approvals at the end of sync right # before we commit the transaction to avoid race conditions. with connection.cursor() as cursor: cursor.execute(""" UPDATE base_translation AS b SET approved = FALSE, approved_date = NULL WHERE id IN (SELECT trans.id FROM base_translation AS trans LEFT JOIN base_entity AS ent ON ent.id = trans.entity_id LEFT JOIN base_resource AS res ON res.id = ent.resource_id WHERE locale_id = %(locale_id)s AND res.project_id = %(project_id)s) AND approved_date != (SELECT max(approved_date) FROM base_translation WHERE entity_id = b.entity_id AND locale_id = b.locale_id AND (plural_form = b.plural_form OR plural_form IS NULL)); """, { 'locale_id': locale.id, 'project_id': db_project.id }) # Perform the commit last so that, if it succeeds, there is # nothing after it to fail. if not no_commit and locale in changeset.locales_to_commit: commit_changes(db_project, vcs_project, changeset, locale) except CommitToRepositoryException as err: # Transaction aborted, log and move on to the next locale. log.warning( 'Failed to sync locale {locale} for project {project} due to ' 'commit error: {error}'.format( locale=locale.code, project=db_project.slug, error=err, ) ) with transaction.atomic(): db_project.aggregate_stats() log.info('Synced translations for project {0} in locales {1}.'.format( db_project.slug, ','.join(locale.code for locale in vcs_project.synced_locales) )) end_repo_sync(repo, repo_sync_log)
def sync_translations(self, project_pk, repo_pk, project_sync_log_pk, now, project_changes=None, obsolete_vcs_resources=None, new_paths=None, locale=None, no_pull=False, no_commit=False, full_scan=False): db_project = get_or_fail(Project, pk=project_pk, message='Could not sync project with pk={0}, not found.'.format(project_pk)) repo = get_or_fail(Repository, pk=repo_pk, message='Could not sync repo with pk={0}, not found.'.format(repo_pk)) project_sync_log = get_or_fail(ProjectSyncLog, pk=project_sync_log_pk, message=('Could not sync project {0}, log with pk={1} not found.' .format(db_project.slug, project_sync_log_pk))) log.info('Syncing translations for repo: {}'.format(repo.url)) repo_sync_log = RepositorySyncLog.objects.create( project_sync_log=project_sync_log, repository=repo, start_time=timezone.now() ) if locale: locales = [locale] else: locales = repo.locales if not locales: log.info('Skipping repo `{0}` for project {1}, no locales to sync found within.' .format(repo.url, db_project.slug)) repo_sync_log.end() return # Pull VCS changes in case we're on a different worker than the one # sync started on. if not no_pull: repos_changed = pull_changes(db_project) resources_changed = [] obsolete_vcs_entities = [] if project_changes: resources_changed = ( project_changes['update_db'] + project_changes['obsolete_db'] + project_changes['create_db'] ) obsolete_vcs_entities = project_changes['obsolete_db'] # If none of the repos has changed since the last sync and there are # no Pontoon-side changes for this project, quit early. if not full_scan and not db_project.needs_sync and not repos_changed and not (resources_changed or obsolete_vcs_resources): log.info('Skipping project {0}, no changes detected.'.format(db_project.slug)) repo_sync_log.end() return obsolete_entities_paths = Resource.objects.obsolete_entities_paths(obsolete_vcs_entities) if obsolete_vcs_entities else None vcs_project = VCSProject( db_project, locales=locales, obsolete_entities_paths=obsolete_entities_paths, new_paths=new_paths, full_scan=full_scan ) failed_locales = set() for locale in locales: try: with transaction.atomic(): # Sets VCSProject.synced_locales, needed to skip early if not vcs_project.synced_locales: vcs_project.resources if not obsolete_vcs_resources: # Skip all locales if none of the them has anything to sync if len(vcs_project.synced_locales) == 0: if resources_changed: for l in locales: update_translated_resources(db_project, vcs_project, l) update_locale_project_locale_stats(l, db_project) db_project.aggregate_stats() log.info('Skipping repo `{0}` for project {1}, none of the locales has anything to sync.' .format(repo.url, db_project.slug)) repo.set_last_synced_revisions() repo_sync_log.end() return # Skip locales that have nothing to sync if vcs_project.synced_locales and locale not in vcs_project.synced_locales: if resources_changed: update_translated_resources(db_project, vcs_project, locale) update_locale_project_locale_stats(locale, db_project) log.debug('Skipping locale `{0}` for project {1}, no changes detected.' .format(locale.code, db_project.slug)) continue changeset = ChangeSet(db_project, vcs_project, now, obsolete_vcs_entities, obsolete_vcs_resources, locale) update_translations(db_project, vcs_project, locale, changeset) changeset.execute() update_translated_resources(db_project, vcs_project, locale) update_locale_project_locale_stats(locale, db_project) # Clear out the "has_changed" markers now that we've finished # syncing. (ChangedEntityLocale.objects .filter(entity__resource__project=db_project, locale=locale, when__lte=now) .delete()) # Clean up any duplicate approvals at the end of sync right # before we commit the transaction to avoid race conditions. with connection.cursor() as cursor: cursor.execute(""" UPDATE base_translation AS b SET approved = FALSE, approved_date = NULL WHERE id IN (SELECT trans.id FROM base_translation AS trans LEFT JOIN base_entity AS ent ON ent.id = trans.entity_id LEFT JOIN base_resource AS res ON res.id = ent.resource_id WHERE locale_id = %(locale_id)s AND res.project_id = %(project_id)s) AND approved_date != (SELECT max(approved_date) FROM base_translation WHERE entity_id = b.entity_id AND locale_id = b.locale_id AND (plural_form = b.plural_form OR plural_form IS NULL)); """, { 'locale_id': locale.id, 'project_id': db_project.id }) # Perform the commit last so that, if it succeeds, there is # nothing after it to fail. if not no_commit and locale in changeset.locales_to_commit: commit_changes(db_project, vcs_project, changeset, locale) log.info( 'Synced locale {locale} for project {project}.'.format( locale=locale.code, project=db_project.slug, ) ) except CommitToRepositoryException as err: # Transaction aborted, log and move on to the next locale. log.warning( 'Failed to sync locale {locale} for project {project} due to ' 'commit error: {error}'.format( locale=locale.code, project=db_project.slug, error=err, ) ) failed_locales.add(locale) with transaction.atomic(): db_project.aggregate_stats() synced_locales = [locale.code for locale in (vcs_project.synced_locales - failed_locales)] if synced_locales: log.info('Synced translations for project {0} in locales {1}.'.format( db_project.slug, ','.join(synced_locales) )) else: log.info('Failed to sync translations for project {0} due to commit error.'.format( db_project.slug )) repo.set_last_synced_revisions(exclude=failed_locales) repo_sync_log.end()
def sync_translations( self, project_pk, project_sync_log_pk, now, added_paths=None, removed_paths=None, changed_paths=None, new_entities=None, locale=None, no_pull=False, no_commit=False, full_scan=False, ): db_project = get_or_fail( Project, pk=project_pk, message="Could not sync project with pk={0}, not found.".format( project_pk), ) repos = db_project.translation_repositories() repo_pk = repos[0].pk repo = get_or_fail( Repository, pk=repo_pk, message="Could not sync repo with pk={0}, not found.".format(repo_pk), ) project_sync_log = get_or_fail( ProjectSyncLog, pk=project_sync_log_pk, message=( "Could not sync project {0}, log with pk={1} not found.".format( db_project.slug, project_sync_log_pk)), ) log.info("Syncing translations for project: {}".format(db_project.slug)) repo_sync_log = RepositorySyncLog.objects.create( project_sync_log=project_sync_log, repository=repo, start_time=timezone.now()) if locale: locales = db_project.locales.filter(pk=locale.pk) else: locales = db_project.locales.all() if not locales: log.info( "Skipping syncing translations for project {0}, no locales to sync " "found within.".format(db_project.slug)) repo_sync_log.end() return # If project repositories have API access, we can retrieve latest commit hashes and detect # changed locales before the expensive VCS pull/clone operations. When performing full scan, # we still need to sync all locales. if not full_scan: locales = get_changed_locales(db_project, locales, now) readonly_locales = db_project.locales.filter(project_locale__readonly=True) added_and_changed_resources = db_project.resources.filter( path__in=list(added_paths or []) + list(changed_paths or [])).distinct() # We should also sync files for which source file change - but only for read-only locales. # See bug 1372151 for more details. if added_and_changed_resources: changed_locales_pks = [l.pk for l in locales] readonly_locales_pks = [l.pk for l in readonly_locales] locales = db_project.locales.filter(pk__in=changed_locales_pks + readonly_locales_pks) # Pull VCS changes in case we're on a different worker than the one # sync started on. if not no_pull: log.info("Pulling changes for project {0} started.".format( db_project.slug)) repos_changed, repo_locales = pull_changes(db_project, locales) repos = repos.filter(pk__in=repo_locales.keys()) log.info("Pulling changes for project {0} complete.".format( db_project.slug)) # If none of the repos has changed since the last sync and there are # no Pontoon-side changes for this project, quit early. if (not full_scan and not db_project.needs_sync and not repos_changed and not (added_paths or removed_paths or changed_paths)): log.info("Skipping project {0}, no changes detected.".format( db_project.slug)) repo_sync_log.end() return vcs_project = VCSProject( db_project, now, locales=locales, repo_locales=repo_locales, added_paths=added_paths, changed_paths=changed_paths, full_scan=full_scan, ) synced_locales = set() failed_locales = set() # Store newly added locales and locales with newly added resources new_locales = [] for locale in locales: try: with transaction.atomic(): # Sets VCSProject.synced_locales, needed to skip early if not vcs_project.synced_locales: vcs_project.resources # Skip all locales if none of the them has anything to sync if len(vcs_project.synced_locales) == 0: break # Skip locales that have nothing to sync if (vcs_project.synced_locales and locale not in vcs_project.synced_locales): continue changeset = ChangeSet(db_project, vcs_project, now, locale) update_translations(db_project, vcs_project, locale, changeset) changeset.execute() created = update_translated_resources(db_project, vcs_project, locale) if created: new_locales.append(locale.pk) update_locale_project_locale_stats(locale, db_project) # Clear out the "has_changed" markers now that we've finished # syncing. (ChangedEntityLocale.objects.filter( entity__resource__project=db_project, locale=locale, when__lte=now, ).delete()) # Perform the commit last so that, if it succeeds, there is # nothing after it to fail. if (not no_commit and locale in changeset.locales_to_commit and locale not in readonly_locales): commit_changes(db_project, vcs_project, changeset, locale) log.info( "Synced locale {locale} for project {project}.".format( locale=locale.code, project=db_project.slug, )) synced_locales.add(locale.code) except CommitToRepositoryException as err: # Transaction aborted, log and move on to the next locale. log.warning( "Failed to sync locale {locale} for project {project} due to " "commit error: {error}".format( locale=locale.code, project=db_project.slug, error=err, )) failed_locales.add(locale.code) # If sources have changed, update stats for all locales. if added_paths or removed_paths or changed_paths: for locale in db_project.locales.all(): # Already synced. if locale.code in synced_locales: continue # We have files: update all translated resources. if locale in locales: created = update_translated_resources(db_project, vcs_project, locale) if created: new_locales.append[locale.pk] # We don't have files: we can still update asymmetric translated resources. else: update_translated_resources_no_files( db_project, locale, added_and_changed_resources, ) update_locale_project_locale_stats(locale, db_project) synced_locales.add(locale.code) log.info( "Synced source changes for locale {locale} for project {project}." .format( locale=locale.code, project=db_project.slug, )) db_project.aggregate_stats() if synced_locales: log.info("Synced translations for project {0} in locales {1}.".format( db_project.slug, ",".join(synced_locales))) elif failed_locales: log.info( "Failed to sync translations for project {0} due to commit error.". format(db_project.slug)) else: log.info( "Skipping syncing translations for project {0}, none of the locales " "has anything to sync.".format(db_project.slug)) for r in repos: r.set_last_synced_revisions(locales=repo_locales[r.pk].exclude( code__in=failed_locales)) repo_sync_log.end() if db_project.pretranslation_enabled: # Pretranslate all entities for newly added locales # and locales with newly added resources if len(new_locales): pretranslate(db_project, locales=new_locales) locales = db_project.locales.exclude(pk__in=new_locales).values_list( "pk", flat=True) # Pretranslate newly added entities for all locales if new_entities and locales: new_entities = list(set(new_entities)) pretranslate(db_project, locales=locales, entities=new_entities)
def sync_translations( self, project_pk, project_sync_log_pk, now, project_changes=None, obsolete_vcs_resources=None, new_paths=None, locale=None, no_pull=False, no_commit=False, full_scan=False ): db_project = get_or_fail( Project, pk=project_pk, message='Could not sync project with pk={0}, not found.'.format(project_pk) ) repos = db_project.translation_repositories() repo_pk = repos[0].pk repo = get_or_fail( Repository, pk=repo_pk, message='Could not sync repo with pk={0}, not found.'.format(repo_pk) ) project_sync_log = get_or_fail( ProjectSyncLog, pk=project_sync_log_pk, message=( 'Could not sync project {0}, log with pk={1} not found.' .format(db_project.slug, project_sync_log_pk) ) ) log.info('Syncing translations for project: {}'.format(db_project.slug)) repo_sync_log = RepositorySyncLog.objects.create( project_sync_log=project_sync_log, repository=repo, start_time=timezone.now() ) if locale: locales = db_project.locales.filter(pk=locale.pk) else: locales = db_project.locales.all() if not locales: log.info('Skipping syncing translations for project {0}, no locales to sync found within.' .format(db_project.slug)) repo_sync_log.end() return # If project repositories have API access, we can retrieve latest commit hashes and detect # changed locales before the expensive VCS pull/clone operations. When performing full scan, # we still need to sync all locales. if not full_scan: locales = get_changed_locales(db_project, locales, now) # Pull VCS changes in case we're on a different worker than the one # sync started on. if not no_pull: log.info('Pulling changes for project {0} started.'.format(db_project.slug)) repos_changed, repo_locales = pull_changes(db_project, locales) repos = repos.filter(pk__in=repo_locales.keys()) log.info('Pulling changes for project {0} complete.'.format(db_project.slug)) changed_resources = [] obsolete_vcs_entities = [] if project_changes: updated_entity_pks = [] for locale_code, db_entity, vcs_entity in project_changes['update_db']: updated_entity_pks.append(db_entity.pk) obsolete_entity_pks = project_changes['obsolete_db'] changed_resources = db_project.resources.filter( Q(entities__date_created=now) | Q(entities__pk__in=updated_entity_pks + obsolete_entity_pks) ).distinct() obsolete_vcs_entities = project_changes['obsolete_db'] # If none of the repos has changed since the last sync and there are # no Pontoon-side changes for this project, quit early. if ( not full_scan and not db_project.needs_sync and not repos_changed and not (changed_resources or obsolete_vcs_resources) ): log.info('Skipping project {0}, no changes detected.'.format(db_project.slug)) repo_sync_log.end() return obsolete_entities_paths = ( Resource.objects.obsolete_entities_paths(obsolete_vcs_entities) if obsolete_vcs_entities else None ) vcs_project = VCSProject( db_project, now, locales=locales, repo_locales=repo_locales, obsolete_entities_paths=obsolete_entities_paths, new_paths=new_paths, full_scan=full_scan ) synced_locales = set() failed_locales = set() readonly_locales = db_project.locales.filter(project_locale__readonly=True) for locale in locales: try: with transaction.atomic(): # Sets VCSProject.synced_locales, needed to skip early if not vcs_project.synced_locales: vcs_project.resources # Skip all locales if none of the them has anything to sync if len(vcs_project.synced_locales) == 0: break # Skip locales that have nothing to sync if vcs_project.synced_locales and locale not in vcs_project.synced_locales: continue changeset = ChangeSet(db_project, vcs_project, now, locale) update_translations(db_project, vcs_project, locale, changeset) changeset.execute() update_translated_resources(db_project, vcs_project, locale) update_locale_project_locale_stats(locale, db_project) # Clear out the "has_changed" markers now that we've finished # syncing. (ChangedEntityLocale.objects .filter(entity__resource__project=db_project, locale=locale, when__lte=now) .delete()) # Clean up any duplicate approvals at the end of sync right # before we commit the transaction to avoid race conditions. with connection.cursor() as cursor: cursor.execute(""" UPDATE base_translation AS b SET approved = FALSE, approved_date = NULL WHERE id IN (SELECT trans.id FROM base_translation AS trans LEFT JOIN base_entity AS ent ON ent.id = trans.entity_id LEFT JOIN base_resource AS res ON res.id = ent.resource_id WHERE locale_id = %(locale_id)s AND res.project_id = %(project_id)s) AND approved_date != (SELECT max(approved_date) FROM base_translation WHERE entity_id = b.entity_id AND locale_id = b.locale_id AND (plural_form = b.plural_form OR plural_form IS NULL)); """, { 'locale_id': locale.id, 'project_id': db_project.id }) # Perform the commit last so that, if it succeeds, there is # nothing after it to fail. if ( not no_commit and locale in changeset.locales_to_commit and locale not in readonly_locales ): commit_changes(db_project, vcs_project, changeset, locale) log.info( 'Synced locale {locale} for project {project}.'.format( locale=locale.code, project=db_project.slug, ) ) synced_locales.add(locale.code) except CommitToRepositoryException as err: # Transaction aborted, log and move on to the next locale. log.warning( 'Failed to sync locale {locale} for project {project} due to ' 'commit error: {error}'.format( locale=locale.code, project=db_project.slug, error=err, ) ) failed_locales.add(locale.code) # If sources have changed, update stats for all locales. if changed_resources or obsolete_vcs_resources: for locale in db_project.locales.all(): # Already synced. if locale.code in synced_locales: continue # We have files: update all translated resources. if locale in locales: update_translated_resources(db_project, vcs_project, locale) # We don't have files: we can still update asymmetric translated resources. else: update_translated_resources_no_files(db_project, locale, changed_resources) update_locale_project_locale_stats(locale, db_project) synced_locales.add(locale.code) log.info( 'Synced source changes for locale {locale} for project {project}.'.format( locale=locale.code, project=db_project.slug, ) ) db_project.aggregate_stats() if synced_locales: log.info('Synced translations for project {0} in locales {1}.'.format( db_project.slug, ','.join(synced_locales) )) elif failed_locales: log.info('Failed to sync translations for project {0} due to commit error.'.format( db_project.slug )) else: log.info( 'Skipping syncing translations for project {0}, none of the locales ' 'has anything to sync.'.format(db_project.slug) ) for r in repos: r.set_last_synced_revisions( locales=repo_locales[r.pk].exclude(code__in=failed_locales) ) repo_sync_log.end()
def sync_translations( self, project_pk, project_sync_log_pk, now, added_paths=None, removed_paths=None, changed_paths=None, locale=None, no_pull=False, no_commit=False, full_scan=False, ): db_project = get_or_fail( Project, pk=project_pk, message='Could not sync project with pk={0}, not found.'.format(project_pk) ) repos = db_project.translation_repositories() repo_pk = repos[0].pk repo = get_or_fail( Repository, pk=repo_pk, message='Could not sync repo with pk={0}, not found.'.format(repo_pk) ) project_sync_log = get_or_fail( ProjectSyncLog, pk=project_sync_log_pk, message=( 'Could not sync project {0}, log with pk={1} not found.' .format(db_project.slug, project_sync_log_pk) ) ) log.info('Syncing translations for project: {}'.format(db_project.slug)) repo_sync_log = RepositorySyncLog.objects.create( project_sync_log=project_sync_log, repository=repo, start_time=timezone.now() ) if locale: locales = db_project.locales.filter(pk=locale.pk) else: locales = db_project.locales.all() if not locales: log.info('Skipping syncing translations for project {0}, no locales to sync found within.' .format(db_project.slug)) repo_sync_log.end() return # If project repositories have API access, we can retrieve latest commit hashes and detect # changed locales before the expensive VCS pull/clone operations. When performing full scan, # we still need to sync all locales. if not full_scan: locales = get_changed_locales(db_project, locales, now) readonly_locales = db_project.locales.filter(project_locale__readonly=True) added_and_changed_resources = db_project.resources.filter( path__in=list(added_paths or []) + list(changed_paths or []) ).distinct() # We should also sync files for which source file change - but only for read-only locales. # See bug 1372151 for more details. if added_and_changed_resources: changed_locales_pks = [l.pk for l in locales] readonly_locales_pks = [l.pk for l in readonly_locales] locales = db_project.locales.filter( pk__in=changed_locales_pks + readonly_locales_pks ) # Pull VCS changes in case we're on a different worker than the one # sync started on. if not no_pull: log.info('Pulling changes for project {0} started.'.format(db_project.slug)) repos_changed, repo_locales = pull_changes(db_project, locales) repos = repos.filter(pk__in=repo_locales.keys()) log.info('Pulling changes for project {0} complete.'.format(db_project.slug)) # If none of the repos has changed since the last sync and there are # no Pontoon-side changes for this project, quit early. if ( not full_scan and not db_project.needs_sync and not repos_changed and not (added_paths or removed_paths or changed_paths) ): log.info('Skipping project {0}, no changes detected.'.format(db_project.slug)) repo_sync_log.end() return vcs_project = VCSProject( db_project, now, locales=locales, repo_locales=repo_locales, added_paths=added_paths, changed_paths=changed_paths, full_scan=full_scan ) synced_locales = set() failed_locales = set() for locale in locales: try: with transaction.atomic(): # Sets VCSProject.synced_locales, needed to skip early if not vcs_project.synced_locales: vcs_project.resources # Skip all locales if none of the them has anything to sync if len(vcs_project.synced_locales) == 0: break # Skip locales that have nothing to sync if vcs_project.synced_locales and locale not in vcs_project.synced_locales: continue changeset = ChangeSet(db_project, vcs_project, now, locale) update_translations(db_project, vcs_project, locale, changeset) changeset.execute() update_translated_resources(db_project, vcs_project, locale) update_locale_project_locale_stats(locale, db_project) # Clear out the "has_changed" markers now that we've finished # syncing. (ChangedEntityLocale.objects .filter(entity__resource__project=db_project, locale=locale, when__lte=now) .delete()) # Perform the commit last so that, if it succeeds, there is # nothing after it to fail. if ( not no_commit and locale in changeset.locales_to_commit and locale not in readonly_locales ): commit_changes(db_project, vcs_project, changeset, locale) log.info( 'Synced locale {locale} for project {project}.'.format( locale=locale.code, project=db_project.slug, ) ) synced_locales.add(locale.code) except CommitToRepositoryException as err: # Transaction aborted, log and move on to the next locale. log.warning( 'Failed to sync locale {locale} for project {project} due to ' 'commit error: {error}'.format( locale=locale.code, project=db_project.slug, error=err, ) ) failed_locales.add(locale.code) # If sources have changed, update stats for all locales. if added_paths or removed_paths or changed_paths: for locale in db_project.locales.all(): # Already synced. if locale.code in synced_locales: continue # We have files: update all translated resources. if locale in locales: update_translated_resources(db_project, vcs_project, locale) # We don't have files: we can still update asymmetric translated resources. else: update_translated_resources_no_files( db_project, locale, added_and_changed_resources, ) update_locale_project_locale_stats(locale, db_project) synced_locales.add(locale.code) log.info( 'Synced source changes for locale {locale} for project {project}.'.format( locale=locale.code, project=db_project.slug, ) ) db_project.aggregate_stats() if synced_locales: log.info('Synced translations for project {0} in locales {1}.'.format( db_project.slug, ','.join(synced_locales) )) elif failed_locales: log.info('Failed to sync translations for project {0} due to commit error.'.format( db_project.slug )) else: log.info( 'Skipping syncing translations for project {0}, none of the locales ' 'has anything to sync.'.format(db_project.slug) ) for r in repos: r.set_last_synced_revisions( locales=repo_locales[r.pk].exclude(code__in=failed_locales) ) repo_sync_log.end()
def sync_translations( self, project_pk, project_sync_log_pk, now, project_changes=None, obsolete_vcs_resources=None, new_paths=None, locale=None, no_pull=False, no_commit=False, full_scan=False ): db_project = get_or_fail( Project, pk=project_pk, message='Could not sync project with pk={0}, not found.'.format(project_pk) ) repos = db_project.translation_repositories() repo_pk = repos[0].pk repo = get_or_fail( Repository, pk=repo_pk, message='Could not sync repo with pk={0}, not found.'.format(repo_pk) ) project_sync_log = get_or_fail( ProjectSyncLog, pk=project_sync_log_pk, message=( 'Could not sync project {0}, log with pk={1} not found.' .format(db_project.slug, project_sync_log_pk) ) ) log.info('Syncing translations for project: {}'.format(db_project.slug)) repo_sync_log = RepositorySyncLog.objects.create( project_sync_log=project_sync_log, repository=repo, start_time=timezone.now() ) if locale: locales = db_project.locales.filter(pk=locale.pk) else: locales = db_project.locales.all() if not locales: log.info('Skipping syncing translations for project {0}, no locales to sync found within.' .format(db_project.slug)) repo_sync_log.end() return # If project repositories have API access, we can retrieve latest commit hashes and detect # changed locales before the expensive VCS pull/clone operations. When performing full scan, # we still need to sync all locales. if not full_scan: locales = get_changed_locales(db_project, locales, now) # Pull VCS changes in case we're on a different worker than the one # sync started on. if not no_pull: log.info('Pulling changes for project {0} started.'.format(db_project.slug)) repos_changed, repo_locales = pull_changes(db_project, locales) repos = repos.filter(pk__in=repo_locales.keys()) log.info('Pulling changes for project {0} complete.'.format(db_project.slug)) changed_resources = [] obsolete_vcs_entities = [] if project_changes: updated_entity_pks = [] for locale_code, db_entity, vcs_entity in project_changes['update_db']: updated_entity_pks.append(db_entity.pk) obsolete_entity_pks = project_changes['obsolete_db'] changed_resources = db_project.resources.filter( Q(entities__date_created=now) | Q(entities__pk__in=updated_entity_pks + obsolete_entity_pks) ).distinct() obsolete_vcs_entities = project_changes['obsolete_db'] # If none of the repos has changed since the last sync and there are # no Pontoon-side changes for this project, quit early. if ( not full_scan and not db_project.needs_sync and not repos_changed and not (changed_resources or obsolete_vcs_resources) ): log.info('Skipping project {0}, no changes detected.'.format(db_project.slug)) repo_sync_log.end() return obsolete_entities_paths = ( Resource.objects.obsolete_entities_paths(obsolete_vcs_entities) if obsolete_vcs_entities else None ) vcs_project = VCSProject( db_project, now, locales=locales, repo_locales=repo_locales, obsolete_entities_paths=obsolete_entities_paths, new_paths=new_paths, full_scan=full_scan ) synced_locales = set() failed_locales = set() for locale in locales: try: with transaction.atomic(): # Sets VCSProject.synced_locales, needed to skip early if not vcs_project.synced_locales: vcs_project.resources # Skip all locales if none of the them has anything to sync if len(vcs_project.synced_locales) == 0: break # Skip locales that have nothing to sync if vcs_project.synced_locales and locale not in vcs_project.synced_locales: continue changeset = ChangeSet(db_project, vcs_project, now, locale) update_translations(db_project, vcs_project, locale, changeset) changeset.execute() update_translated_resources(db_project, vcs_project, locale) update_locale_project_locale_stats(locale, db_project) # Clear out the "has_changed" markers now that we've finished # syncing. (ChangedEntityLocale.objects .filter(entity__resource__project=db_project, locale=locale, when__lte=now) .delete()) # Clean up any duplicate approvals at the end of sync right # before we commit the transaction to avoid race conditions. with connection.cursor() as cursor: cursor.execute(""" UPDATE base_translation AS b SET approved = FALSE, approved_date = NULL WHERE id IN (SELECT trans.id FROM base_translation AS trans LEFT JOIN base_entity AS ent ON ent.id = trans.entity_id LEFT JOIN base_resource AS res ON res.id = ent.resource_id WHERE locale_id = %(locale_id)s AND res.project_id = %(project_id)s) AND approved_date != (SELECT max(approved_date) FROM base_translation WHERE entity_id = b.entity_id AND locale_id = b.locale_id AND (plural_form = b.plural_form OR plural_form IS NULL)); """, { 'locale_id': locale.id, 'project_id': db_project.id }) # Perform the commit last so that, if it succeeds, there is # nothing after it to fail. if not no_commit and locale in changeset.locales_to_commit: commit_changes(db_project, vcs_project, changeset, locale) log.info( 'Synced locale {locale} for project {project}.'.format( locale=locale.code, project=db_project.slug, ) ) synced_locales.add(locale.code) except CommitToRepositoryException as err: # Transaction aborted, log and move on to the next locale. log.warning( 'Failed to sync locale {locale} for project {project} due to ' 'commit error: {error}'.format( locale=locale.code, project=db_project.slug, error=err, ) ) failed_locales.add(locale.code) # If sources have changed, update stats for all locales. if changed_resources or obsolete_vcs_resources: for locale in db_project.locales.all(): # Already synced. if locale.code in synced_locales: continue # We have files: update all translated resources. if locale in locales: update_translated_resources(db_project, vcs_project, locale) # We don't have files: we can still update asymmetric translated resources. else: update_translated_resources_no_files(db_project, locale, changed_resources) update_locale_project_locale_stats(locale, db_project) synced_locales.add(locale.code) log.info( 'Synced source changes for locale {locale} for project {project}.'.format( locale=locale.code, project=db_project.slug, ) ) db_project.aggregate_stats() if synced_locales: log.info('Synced translations for project {0} in locales {1}.'.format( db_project.slug, ','.join(synced_locales) )) elif failed_locales: log.info('Failed to sync translations for project {0} due to commit error.'.format( db_project.slug )) else: log.info( 'Skipping syncing translations for project {0}, none of the locales ' 'has anything to sync.'.format(db_project.slug) ) for r in repos: r.set_last_synced_revisions( locales=repo_locales[r.pk].exclude(code__in=failed_locales) ) repo_sync_log.end()