def test_unchanged(self): """ If the revisions returned by repo.pull match those from the last sync, consider the VCS unchanged and return False. """ self.mock_repo_pull.return_value = {"single_locale": "asdf"} self.repository.last_synced_revisions = {"single_locale": "asdf"} self.repository.save() has_changed, _ = pull_locale_repo_changes( self.db_project, locales=self.db_project.locales.all()) assert not has_changed
def test_basic(self): """ Pull_changes should call repo.pull for each repo for the project and return whether any changes happened in VCS. """ mock_db_project = MagicMock() mock_db_project.repositories.all.return_value = [self.repository] self.mock_repo_pull.return_value = {"single_locale": "asdf"} has_changed, _ = pull_locale_repo_changes(self.db_project, self.locales) assert has_changed
def test_unsure_changes(self): """ If any of the repos returns None as a revision number, consider the VCS as changed even if the revisions match the last sync. """ self.mock_repo_pull.return_value = {"single_locale": None} self.repository.last_synced_revisions = {"single_locale": None} self.repository.save() has_changed, _ = pull_locale_repo_changes(self.db_project, self.locales) assert has_changed
def sync_translations( db_project, project_sync_log, now, has_source_repo_changed, added_paths=None, removed_paths=None, changed_paths=None, new_entities=None, no_pull=False, no_commit=False, force=False, ): repo = db_project.translation_repositories()[0] log.info("Syncing translations for project: {}".format(db_project.slug)) repo_sync_log = RepositorySyncLog.objects.create( project_sync_log=project_sync_log, repository=repo, start_time=timezone.now() ) locales = db_project.locales.all() if not locales: log.info( "Skipping syncing translations for project {0}, no locales to sync " "found within.".format(db_project.slug) ) repo_sync_log.end() return # If project repositories have API access, we can retrieve latest commit hashes and detect # changed locales before the expensive VCS pull/clone operations. When performing full scan, # we still need to sync all locales. if not force: locales = get_changed_locales(db_project, locales, now) readonly_locales = db_project.locales.filter(project_locale__readonly=True) added_and_changed_resources = db_project.resources.filter( path__in=list(added_paths or []) + list(changed_paths or []) ).distinct() # We should also sync files for which source file change - but only for read-only locales. # See bug 1372151 for more details. if added_and_changed_resources: changed_locales_pks = [l.pk for l in locales] readonly_locales_pks = [l.pk for l in readonly_locales] locales = db_project.locales.filter( pk__in=changed_locales_pks + readonly_locales_pks ) have_repos_changed = has_source_repo_changed repo_locales = None if not no_pull: repo_locales = {db_project.source_repository.pk: Locale.objects.none()} # Pull repos of locales in case of multi_locale_project if not db_project.has_single_repo: log.info( "Pulling locale repos for project {0} started.".format(db_project.slug) ) have_locale_repos_changed, pulled_repo_locales = pull_locale_repo_changes( db_project, locales ) log.info( "Pulling locale repos for project {0} complete.".format(db_project.slug) ) have_repos_changed |= have_locale_repos_changed repo_locales.update(pulled_repo_locales) # If none of the repos has changed since the last sync and there are # no Pontoon-side changes for this project, quit early. if ( not force and not db_project.needs_sync and not have_repos_changed and not (added_paths or removed_paths or changed_paths) ): log.info("Skipping project {0}, no changes detected.".format(db_project.slug)) repo_sync_log.end() return vcs_project = VCSProject( db_project, now, locales=locales, repo_locales=repo_locales, added_paths=added_paths, changed_paths=changed_paths, force=force, ) synced_locales = set() failed_locales = set() # Store newly added locales and locales with newly added resources new_locales = [] for locale in locales: try: with transaction.atomic(): # Sets VCSProject.synced_locales, needed to skip early if not vcs_project.synced_locales: vcs_project.resources # Skip all locales if none of the them has anything to sync if len(vcs_project.synced_locales) == 0: break # Skip locales that have nothing to sync if ( vcs_project.synced_locales and locale not in vcs_project.synced_locales ): continue changeset = ChangeSet(db_project, vcs_project, now, locale) update_translations(db_project, vcs_project, locale, changeset) changeset.execute() created = update_translated_resources(db_project, vcs_project, locale) if created: new_locales.append(locale.pk) update_locale_project_locale_stats(locale, db_project) # Clear out the "has_changed" markers now that we've finished # syncing. ( ChangedEntityLocale.objects.filter( entity__resource__project=db_project, locale=locale, when__lte=now, ).delete() ) # Perform the commit last so that, if it succeeds, there is # nothing after it to fail. if ( not no_commit and locale in changeset.locales_to_commit and locale not in readonly_locales ): commit_changes(db_project, vcs_project, changeset, locale) log.info( "Synced locale {locale} for project {project}.".format( locale=locale.code, project=db_project.slug, ) ) synced_locales.add(locale.code) except CommitToRepositoryException as err: # Transaction aborted, log and move on to the next locale. log.warning( "Failed to sync locale {locale} for project {project} due to " "commit error: {error}".format( locale=locale.code, project=db_project.slug, error=err, ) ) failed_locales.add(locale.code) # If sources have changed, update stats for all locales. if added_paths or removed_paths or changed_paths: for locale in db_project.locales.all(): # Already synced. if locale.code in synced_locales: continue # We have files: update all translated resources. if locale in locales: created = update_translated_resources(db_project, vcs_project, locale) if created: new_locales.append[locale.pk] # We don't have files: we can still update asymmetric translated resources. else: update_translated_resources_no_files( db_project, locale, added_and_changed_resources, ) update_locale_project_locale_stats(locale, db_project) synced_locales.add(locale.code) log.info( "Synced source changes for locale {locale} for project {project}.".format( locale=locale.code, project=db_project.slug, ) ) db_project.aggregate_stats() if synced_locales: log.info( "Synced translations for project {0} in locales {1}.".format( db_project.slug, ",".join(synced_locales) ) ) elif failed_locales: log.info( "Failed to sync translations for project {0} due to commit error.".format( db_project.slug ) ) else: log.info( "Skipping syncing translations for project {0}, none of the locales " "has anything to sync.".format(db_project.slug) ) if repo_locales: repos = db_project.repositories.filter(pk__in=repo_locales.keys()) for r in repos: r.set_last_synced_revisions( locales=repo_locales[r.pk].exclude(code__in=failed_locales) ) repo_sync_log.end() if db_project.pretranslation_enabled: # Pretranslate all entities for newly added locales # and locales with newly added resources if len(new_locales): pretranslate(db_project.pk, locales=new_locales) locales = db_project.locales.exclude(pk__in=new_locales).values_list( "pk", flat=True ) # Pretranslate newly added entities for all locales if new_entities and locales: new_entities = list(set(new_entities)) pretranslate(db_project.pk, locales=locales, entities=new_entities)