def sync_project(db_project, now): # Only load source resources for updating entities. vcs_project = VCSProject(db_project, locales=[]) with transaction.atomic(): update_resources(db_project, vcs_project) changeset = ChangeSet(db_project, vcs_project, now) update_entities(db_project, vcs_project, changeset) changeset.execute()
def sync_project(db_project, now, full_scan=False): vcs_project = VCSProject(db_project, locales=[], full_scan=full_scan) with transaction.atomic(): removed_paths = update_resources(db_project, vcs_project) changeset = ChangeSet(db_project, vcs_project, now) update_entities(db_project, vcs_project, changeset) changeset.execute() return changeset.changes['obsolete_db'], removed_paths
def sync_project(db_project, now): # Only load source resources for updating entities. vcs_project = VCSProject(db_project, locales=[]) with transaction.atomic(): removed_paths = update_resources(db_project, vcs_project) changeset = ChangeSet(db_project, vcs_project, now) update_entities(db_project, vcs_project, changeset) changeset.execute() return changeset.changes['obsolete_db'], removed_paths
def update_originals(db_project, now, full_scan=False): vcs_project = VCSProject(db_project, locales=[], full_scan=full_scan) with transaction.atomic(): added_paths, removed_paths, changed_paths = update_resources(db_project, vcs_project) changeset = ChangeSet(db_project, vcs_project, now) update_entities(db_project, vcs_project, changeset) changeset.execute() return added_paths, removed_paths, changed_paths
def setUp(self): self.now = aware_datetime(1970, 1, 1) timezone_patch = patch('pontoon.sync.tasks.timezone') self.mock_timezone = timezone_patch.start() self.addCleanup(timezone_patch.stop) self.mock_timezone.now.return_value = self.now self.translated_locale = LocaleFactory.create(code='translated-locale') self.inactive_locale = LocaleFactory.create(code='inactive-locale') self.repository = RepositoryFactory() self.db_project = ProjectFactory.create( name='db-project', locales=[self.translated_locale], repositories=[self.repository]) self.main_db_resource = ResourceFactory.create(project=self.db_project, path='main.lang', format='lang') self.other_db_resource = ResourceFactory.create( project=self.db_project, path='other.lang', format='lang') self.missing_db_resource = ResourceFactory.create( project=self.db_project, path='missing.lang', format='lang') self.main_db_entity = EntityFactory.create( resource=self.main_db_resource, string='Source String', key='Source String', obsolete=False) self.other_db_entity = EntityFactory.create( resource=self.other_db_resource, string='Other Source String', key='Other Source String', obsolete=False) self.main_db_translation = TranslationFactory.create( entity=self.main_db_entity, plural_form=None, locale=self.translated_locale, string='Translated String', date=aware_datetime(1970, 1, 1), approved=True, extra={'tags': []}) # Load paths from the fake locale directory. checkout_path_patch = patch.object(Project, 'checkout_path', new_callable=PropertyMock, return_value=FAKE_CHECKOUT_PATH) checkout_path_patch.start() self.addCleanup(checkout_path_patch.stop) self.vcs_project = VCSProject(self.db_project) self.main_vcs_resource = self.vcs_project.resources[ self.main_db_resource.path] self.other_vcs_resource = self.vcs_project.resources[ self.other_db_resource.path] self.missing_vcs_resource = self.vcs_project.resources[ self.missing_db_resource.path] self.main_vcs_entity = self.main_vcs_resource.entities['Source String'] self.main_vcs_translation = self.main_vcs_entity.translations[ 'translated-locale'] # Mock VCSResource.save() for each resource to avoid altering # the filesystem. resource_save_patch = patch.object(VCSResource, 'save') resource_save_patch.start() self.addCleanup(resource_save_patch.stop) self.changeset = ChangeSet(self.db_project, self.vcs_project, aware_datetime(1970, 1, 1))
def handle_upload_content(slug, code, part, f, user): """ Update translations in the database from uploaded file. :param str slug: Project slug. :param str code: Locale code. :param str part: Resource path or Subpage name. :param UploadedFile f: UploadedFile instance. :param User user: User uploading the file. """ # Avoid circular import; someday we should refactor to avoid. from pontoon.sync import formats from pontoon.sync.changeset import ChangeSet from pontoon.sync.vcs.models import VCSProject from pontoon.base.models import ( ChangedEntityLocale, Entity, Locale, Project, Resource, TranslatedResource, Translation, ) relative_path = _get_relative_path_from_part(slug, part) project = get_object_or_404(Project, slug=slug) locale = get_object_or_404(Locale, code__iexact=code) resource = get_object_or_404(Resource, project__slug=slug, path=relative_path) # Store uploaded file to a temporary file and parse it extension = os.path.splitext(f.name)[1] with tempfile.NamedTemporaryFile(suffix=extension) as temp: for chunk in f.chunks(): temp.write(chunk) temp.flush() resource_file = formats.parse(temp.name) # Update database objects from file changeset = ChangeSet( project, VCSProject(project, locales=[locale]), timezone.now() ) entities_qs = Entity.objects.filter( resource__project=project, resource__path=relative_path, obsolete=False ).prefetch_related( Prefetch( 'translation_set', queryset=Translation.objects.filter(locale=locale), to_attr='db_translations' ) ).prefetch_related( Prefetch( 'translation_set', queryset=Translation.objects.filter(locale=locale, approved_date__lte=timezone.now()), to_attr='old_translations' ) ) entities_dict = {entity.key: entity for entity in entities_qs} for vcs_translation in resource_file.translations: key = vcs_translation.key if key in entities_dict: entity = entities_dict[key] changeset.update_entity_translations_from_vcs( entity, locale.code, vcs_translation, user, entity.db_translations, entity.old_translations ) changeset.bulk_create_translations() changeset.bulk_update_translations() TranslatedResource.objects.get(resource=resource, locale=locale).calculate_stats() # Mark translations as changed changed_entities = {} existing = ChangedEntityLocale.objects.values_list('entity', 'locale').distinct() for t in changeset.translations_to_create + changeset.translations_to_update: key = (t.entity.pk, t.locale.pk) # Remove duplicate changes to prevent unique constraint violation if not key in existing: changed_entities[key] = ChangedEntityLocale(entity=t.entity, locale=t.locale) ChangedEntityLocale.objects.bulk_create(changed_entities.values())
def sync_translations( self, project_pk, project_sync_log_pk, now, added_paths=None, removed_paths=None, changed_paths=None, new_entities=None, locale=None, no_pull=False, no_commit=False, full_scan=False, ): db_project = get_or_fail( Project, pk=project_pk, message="Could not sync project with pk={0}, not found.".format(project_pk), ) repos = db_project.translation_repositories() repo_pk = repos[0].pk repo = get_or_fail( Repository, pk=repo_pk, message="Could not sync repo with pk={0}, not found.".format(repo_pk), ) project_sync_log = get_or_fail( ProjectSyncLog, pk=project_sync_log_pk, message=( "Could not sync project {0}, log with pk={1} not found.".format( db_project.slug, project_sync_log_pk ) ), ) log.info("Syncing translations for project: {}".format(db_project.slug)) repo_sync_log = RepositorySyncLog.objects.create( project_sync_log=project_sync_log, repository=repo, start_time=timezone.now() ) if locale: locales = db_project.locales.filter(pk=locale.pk) else: locales = db_project.locales.all() if not locales: log.info( "Skipping syncing translations for project {0}, no locales to sync " "found within.".format(db_project.slug) ) repo_sync_log.end() return # If project repositories have API access, we can retrieve latest commit hashes and detect # changed locales before the expensive VCS pull/clone operations. When performing full scan, # we still need to sync all locales. if not full_scan: locales = get_changed_locales(db_project, locales, now) readonly_locales = db_project.locales.filter(project_locale__readonly=True) added_and_changed_resources = db_project.resources.filter( path__in=list(added_paths or []) + list(changed_paths or []) ).distinct() # We should also sync files for which source file change - but only for read-only locales. # See bug 1372151 for more details. if added_and_changed_resources: changed_locales_pks = [l.pk for l in locales] readonly_locales_pks = [l.pk for l in readonly_locales] locales = db_project.locales.filter( pk__in=changed_locales_pks + readonly_locales_pks ) # Pull VCS changes in case we're on a different worker than the one # sync started on. if not no_pull: log.info("Pulling changes for project {0} started.".format(db_project.slug)) repos_changed, repo_locales = pull_changes(db_project, locales) repos = repos.filter(pk__in=repo_locales.keys()) log.info("Pulling changes for project {0} complete.".format(db_project.slug)) # If none of the repos has changed since the last sync and there are # no Pontoon-side changes for this project, quit early. if ( not full_scan and not db_project.needs_sync and not repos_changed and not (added_paths or removed_paths or changed_paths) ): log.info("Skipping project {0}, no changes detected.".format(db_project.slug)) repo_sync_log.end() return vcs_project = VCSProject( db_project, now, locales=locales, repo_locales=repo_locales, added_paths=added_paths, changed_paths=changed_paths, full_scan=full_scan, ) synced_locales = set() failed_locales = set() # Store newly added locales and locales with newly added resources new_locales = [] for locale in locales: try: with transaction.atomic(): # Sets VCSProject.synced_locales, needed to skip early if not vcs_project.synced_locales: vcs_project.resources # Skip all locales if none of the them has anything to sync if len(vcs_project.synced_locales) == 0: break # Skip locales that have nothing to sync if ( vcs_project.synced_locales and locale not in vcs_project.synced_locales ): continue changeset = ChangeSet(db_project, vcs_project, now, locale) update_translations(db_project, vcs_project, locale, changeset) changeset.execute() created = update_translated_resources(db_project, vcs_project, locale) if created: new_locales.append(locale.pk) update_locale_project_locale_stats(locale, db_project) # Clear out the "has_changed" markers now that we've finished # syncing. ( ChangedEntityLocale.objects.filter( entity__resource__project=db_project, locale=locale, when__lte=now, ).delete() ) # Perform the commit last so that, if it succeeds, there is # nothing after it to fail. if ( not no_commit and locale in changeset.locales_to_commit and locale not in readonly_locales ): commit_changes(db_project, vcs_project, changeset, locale) log.info( "Synced locale {locale} for project {project}.".format( locale=locale.code, project=db_project.slug, ) ) synced_locales.add(locale.code) except CommitToRepositoryException as err: # Transaction aborted, log and move on to the next locale. log.warning( "Failed to sync locale {locale} for project {project} due to " "commit error: {error}".format( locale=locale.code, project=db_project.slug, error=err, ) ) failed_locales.add(locale.code) # If sources have changed, update stats for all locales. if added_paths or removed_paths or changed_paths: for locale in db_project.locales.all(): # Already synced. if locale.code in synced_locales: continue # We have files: update all translated resources. if locale in locales: created = update_translated_resources(db_project, vcs_project, locale) if created: new_locales.append[locale.pk] # We don't have files: we can still update asymmetric translated resources. else: update_translated_resources_no_files( db_project, locale, added_and_changed_resources, ) update_locale_project_locale_stats(locale, db_project) synced_locales.add(locale.code) log.info( "Synced source changes for locale {locale} for project {project}.".format( locale=locale.code, project=db_project.slug, ) ) db_project.aggregate_stats() if synced_locales: log.info( "Synced translations for project {0} in locales {1}.".format( db_project.slug, ",".join(synced_locales) ) ) elif failed_locales: log.info( "Failed to sync translations for project {0} due to commit error.".format( db_project.slug ) ) else: log.info( "Skipping syncing translations for project {0}, none of the locales " "has anything to sync.".format(db_project.slug) ) for r in repos: r.set_last_synced_revisions( locales=repo_locales[r.pk].exclude(code__in=failed_locales) ) repo_sync_log.end() if db_project.pretranslation_enabled: # Pretranslate all entities for newly added locales # and locales with newly added resources if len(new_locales): pretranslate(db_project, locales=new_locales) locales = db_project.locales.exclude(pk__in=new_locales).values_list( "pk", flat=True ) # Pretranslate newly added entities for all locales if new_entities and locales: new_entities = list(set(new_entities)) pretranslate(db_project, locales=locales, entities=new_entities)
def sync_project_repo(self, project_pk, repo_pk, project_sync_log_pk, now, obsolete_vcs_entities=None, obsolete_vcs_resources=None, new_paths=None, locale=None, no_pull=False, no_commit=False, full_scan=False): db_project = get_or_fail( Project, pk=project_pk, message='Could not sync project with pk={0}, not found.'.format( project_pk)) repo = get_or_fail( Repository, pk=repo_pk, message='Could not sync repo with pk={0}, not found.'.format(repo_pk)) project_sync_log = get_or_fail( ProjectSyncLog, pk=project_sync_log_pk, message=( 'Could not sync project {0}, log with pk={1} not found.'.format( db_project.slug, project_sync_log_pk))) log.info('Syncing repo: {}'.format(repo.url)) repo_sync_log = RepositorySyncLog.objects.create( project_sync_log=project_sync_log, repository=repo, start_time=timezone.now()) # Pull VCS changes in case we're on a different worker than the one # sync started on. if not no_pull: pull_changes(db_project) if locale: locales = [locale] else: locales = repo.locales # Cannot skip earlier - repo.locales is only available after pull_changes() if not locales: log.debug( 'Skipping repo `{0}` for project {1}, no locales to sync found within.' .format(repo.url, db_project.slug)) repo_sync_log.end() return vcs_project = VCSProject( db_project, locales=locales, obsolete_entities_paths=Resource.objects.obsolete_entities_paths( obsolete_vcs_entities), new_paths=new_paths, full_scan=full_scan) for locale in locales: try: with transaction.atomic(): # Skip locales that have nothing to sync if vcs_project.synced_locales and locale not in vcs_project.synced_locales: if obsolete_vcs_entities or obsolete_vcs_resources: update_locale_project_locale_stats(locale, db_project) continue changeset = ChangeSet(db_project, vcs_project, now, obsolete_vcs_entities, obsolete_vcs_resources) update_translations(db_project, vcs_project, locale, changeset) changeset.execute() update_translated_resources(db_project, vcs_project, changeset, locale) # Skip if none of the locales has anything to sync # VCSProject.synced_locales is set on a first call to # VCSProject.resources, which is set in # pontoon.sync.core.update_translated_resources() if len(vcs_project.synced_locales) == 0: if obsolete_vcs_entities or obsolete_vcs_resources: for l in locales: update_locale_project_locale_stats(l, db_project) db_project.aggregate_stats() log.info( 'Skipping repo `{0}` for project {1}, none of the locales has anything to sync.' .format(repo.url, db_project.slug)) repo_sync_log.end() return update_locale_project_locale_stats(locale, db_project) # Clear out the "has_changed" markers now that we've finished # syncing. (ChangedEntityLocale.objects.filter( entity__resource__project=db_project, locale=locale, when__lte=now).delete()) # Clean up any duplicate approvals at the end of sync right # before we commit the transaction to avoid race conditions. with connection.cursor() as cursor: cursor.execute( """ UPDATE base_translation AS b SET approved = FALSE, approved_date = NULL WHERE id IN (SELECT trans.id FROM base_translation AS trans LEFT JOIN base_entity AS ent ON ent.id = trans.entity_id LEFT JOIN base_resource AS res ON res.id = ent.resource_id WHERE locale_id = %(locale_id)s AND res.project_id = %(project_id)s) AND approved_date != (SELECT max(approved_date) FROM base_translation WHERE entity_id = b.entity_id AND locale_id = b.locale_id AND (plural_form = b.plural_form OR plural_form IS NULL)); """, { 'locale_id': locale.id, 'project_id': db_project.id }) # Perform the commit last so that, if it succeeds, there is # nothing after it to fail. if not no_commit and locale in changeset.locales_to_commit: commit_changes(db_project, vcs_project, changeset, locale) except CommitToRepositoryException as err: # Transaction aborted, log and move on to the next locale. log.warning( 'Failed to sync locale {locale} for project {project} due to ' 'commit error: {error}'.format( locale=locale.code, project=db_project.slug, error=err, )) with transaction.atomic(): db_project.aggregate_stats() log.info('Synced translations for project {0} in locales {1}.'.format( db_project.slug, ','.join(locale.code for locale in vcs_project.synced_locales))) repo_sync_log.end()
def sync_translations(self, project_pk, project_sync_log_pk, now, project_changes=None, obsolete_vcs_resources=None, new_paths=None, locale=None, no_pull=False, no_commit=False, full_scan=False): db_project = get_or_fail( Project, pk=project_pk, message='Could not sync project with pk={0}, not found.'.format( project_pk)) repos = db_project.translation_repositories() repo_pk = repos[0].pk repo = get_or_fail( Repository, pk=repo_pk, message='Could not sync repo with pk={0}, not found.'.format(repo_pk)) project_sync_log = get_or_fail( ProjectSyncLog, pk=project_sync_log_pk, message=( 'Could not sync project {0}, log with pk={1} not found.'.format( db_project.slug, project_sync_log_pk))) log.info('Syncing translations for project: {}'.format(db_project.slug)) repo_sync_log = RepositorySyncLog.objects.create( project_sync_log=project_sync_log, repository=repo, start_time=timezone.now()) if locale: locales = db_project.locales.filter(pk=locale.pk) else: locales = db_project.locales.all() if not locales: log.info( 'Skipping syncing translations for project {0}, no locales to sync found within.' .format(db_project.slug)) repo_sync_log.end() return # If project repositories have API access, we can retrieve latest commit hashes and detect # changed locales before the expensive VCS pull/clone operations. When performing full scan, # we still need to sync all locales. if not full_scan: locales = get_changed_locales(db_project, locales, now) # Pull VCS changes in case we're on a different worker than the one # sync started on. if not no_pull: log.info('Pulling changes for project {0} started.'.format( db_project.slug)) repos_changed, repo_locales = pull_changes(db_project, locales) repos = repos.filter(pk__in=repo_locales.keys()) log.info('Pulling changes for project {0} complete.'.format( db_project.slug)) changed_resources = [] obsolete_vcs_entities = [] if project_changes: updated_entity_pks = [] for locale_code, db_entity, vcs_entity in project_changes['update_db']: updated_entity_pks.append(db_entity.pk) obsolete_entity_pks = project_changes['obsolete_db'] changed_resources = db_project.resources.filter( Q(entities__date_created=now) | Q(entities__pk__in=updated_entity_pks + obsolete_entity_pks)).distinct() obsolete_vcs_entities = project_changes['obsolete_db'] # If none of the repos has changed since the last sync and there are # no Pontoon-side changes for this project, quit early. if (not full_scan and not db_project.needs_sync and not repos_changed and not (changed_resources or obsolete_vcs_resources)): log.info('Skipping project {0}, no changes detected.'.format( db_project.slug)) repo_sync_log.end() return obsolete_entities_paths = ( Resource.objects.obsolete_entities_paths(obsolete_vcs_entities) if obsolete_vcs_entities else None) vcs_project = VCSProject(db_project, now, locales=locales, repo_locales=repo_locales, obsolete_entities_paths=obsolete_entities_paths, new_paths=new_paths, full_scan=full_scan) synced_locales = set() failed_locales = set() for locale in locales: try: with transaction.atomic(): # Sets VCSProject.synced_locales, needed to skip early if not vcs_project.synced_locales: vcs_project.resources # Skip all locales if none of the them has anything to sync if len(vcs_project.synced_locales) == 0: break # Skip locales that have nothing to sync if vcs_project.synced_locales and locale not in vcs_project.synced_locales: continue changeset = ChangeSet(db_project, vcs_project, now, locale) update_translations(db_project, vcs_project, locale, changeset) changeset.execute() update_translated_resources(db_project, vcs_project, locale) update_locale_project_locale_stats(locale, db_project) # Clear out the "has_changed" markers now that we've finished # syncing. (ChangedEntityLocale.objects.filter( entity__resource__project=db_project, locale=locale, when__lte=now).delete()) # Clean up any duplicate approvals at the end of sync right # before we commit the transaction to avoid race conditions. with connection.cursor() as cursor: cursor.execute( """ UPDATE base_translation AS b SET approved = FALSE, approved_date = NULL WHERE id IN (SELECT trans.id FROM base_translation AS trans LEFT JOIN base_entity AS ent ON ent.id = trans.entity_id LEFT JOIN base_resource AS res ON res.id = ent.resource_id WHERE locale_id = %(locale_id)s AND res.project_id = %(project_id)s) AND approved_date != (SELECT max(approved_date) FROM base_translation WHERE entity_id = b.entity_id AND locale_id = b.locale_id AND (plural_form = b.plural_form OR plural_form IS NULL)); """, { 'locale_id': locale.id, 'project_id': db_project.id }) # Perform the commit last so that, if it succeeds, there is # nothing after it to fail. if not no_commit and locale in changeset.locales_to_commit: commit_changes(db_project, vcs_project, changeset, locale) log.info( 'Synced locale {locale} for project {project}.'.format( locale=locale.code, project=db_project.slug, )) synced_locales.add(locale.code) except CommitToRepositoryException as err: # Transaction aborted, log and move on to the next locale. log.warning( 'Failed to sync locale {locale} for project {project} due to ' 'commit error: {error}'.format( locale=locale.code, project=db_project.slug, error=err, )) failed_locales.add(locale.code) # If sources have changed, update stats for all locales. if changed_resources or obsolete_vcs_resources: for locale in db_project.locales.all(): # Already synced. if locale.code in synced_locales: continue # We have files: update all translated resources. if locale in locales: update_translated_resources(db_project, vcs_project, locale) # We don't have files: we can still update asymmetric translated resources. else: update_translated_resources_no_files(db_project, locale, changed_resources) update_locale_project_locale_stats(locale, db_project) synced_locales.add(locale.code) log.info( 'Synced source changes for locale {locale} for project {project}.' .format( locale=locale.code, project=db_project.slug, )) db_project.aggregate_stats() if synced_locales: log.info('Synced translations for project {0} in locales {1}.'.format( db_project.slug, ','.join(synced_locales))) elif failed_locales: log.info( 'Failed to sync translations for project {0} due to commit error.'. format(db_project.slug)) else: log.info( 'Skipping syncing translations for project {0}, none of the locales ' 'has anything to sync.'.format(db_project.slug)) for r in repos: r.set_last_synced_revisions(locales=repo_locales[r.pk].exclude( code__in=failed_locales)) repo_sync_log.end()
def handle_upload_content(slug, code, part, f, user): """ Update translations in the database from uploaded file. :arg str slug: Project slug. :arg str code: Locale code. :arg str part: Resource path or Subpage name. :arg UploadedFile f: UploadedFile instance. :arg User user: User uploading the file. """ # Avoid circular import; someday we should refactor to avoid. from pontoon.sync import formats from pontoon.sync.changeset import ChangeSet from pontoon.sync.vcs.models import VCSProject from pontoon.base.models import ( ChangedEntityLocale, Entity, Locale, Project, Resource, TranslatedResource, Translation, ) relative_path = _get_relative_path_from_part(slug, part) project = get_object_or_404(Project, slug=slug) locale = get_object_or_404(Locale, code=code) resource = get_object_or_404(Resource, project__slug=slug, path=relative_path) # Store uploaded file to a temporary file and parse it extension = os.path.splitext(f.name)[1] with tempfile.NamedTemporaryFile( prefix="strings" if extension == ".xml" else "", suffix=extension, ) as temp: for chunk in f.chunks(): temp.write(chunk) temp.flush() resource_file = formats.parse(temp.name) # Update database objects from file changeset = ChangeSet(project, VCSProject(project, locales=[locale]), timezone.now()) entities_qs = (Entity.objects.filter( resource__project=project, resource__path=relative_path, obsolete=False).prefetch_related( Prefetch( "translation_set", queryset=Translation.objects.filter(locale=locale), to_attr="db_translations", )).prefetch_related( Prefetch( "translation_set", queryset=Translation.objects.filter( locale=locale, approved_date__lte=timezone.now()), to_attr="db_translations_approved_before_sync", ))) entities_dict = {entity.key: entity for entity in entities_qs} for vcs_translation in resource_file.translations: key = vcs_translation.key if key in entities_dict: entity = entities_dict[key] changeset.update_entity_translations_from_vcs( entity, locale.code, vcs_translation, user, entity.db_translations, entity.db_translations_approved_before_sync, ) changeset.bulk_create_translations() changeset.bulk_update_translations() changeset.bulk_log_actions() if changeset.changed_translations: # Update 'active' status of all changed translations and their siblings, # i.e. translations of the same entity to the same locale. changed_pks = {t.pk for t in changeset.changed_translations} (Entity.objects.filter( translation__pk__in=changed_pks).reset_active_translations( locale=locale)) # Run checks and create TM entries for translations that pass them valid_translations = changeset.bulk_check_translations() changeset.bulk_create_translation_memory_entries(valid_translations) # Remove any TM entries of translations that got rejected changeset.bulk_remove_translation_memory_entries() TranslatedResource.objects.get(resource=resource, locale=locale).calculate_stats() # Mark translations as changed changed_entities = {} existing = ChangedEntityLocale.objects.values_list("entity", "locale").distinct() for t in changeset.changed_translations: key = (t.entity.pk, t.locale.pk) # Remove duplicate changes to prevent unique constraint violation if key not in existing: changed_entities[key] = ChangedEntityLocale(entity=t.entity, locale=t.locale) ChangedEntityLocale.objects.bulk_create(changed_entities.values()) # Update latest translation if changeset.translations_to_create: changeset.translations_to_create[-1].update_latest_translation()
def setUp(self): self.now = aware_datetime(1970, 1, 1) timezone_patch = patch("pontoon.sync.tasks.timezone") self.mock_timezone = timezone_patch.start() self.addCleanup(timezone_patch.stop) self.mock_timezone.now.return_value = self.now self.translated_locale = LocaleFactory.create(code="translated-locale") self.inactive_locale = LocaleFactory.create(code="inactive-locale") self.repository = RepositoryFactory() self.db_project = ProjectFactory.create( name="db-project", locales=[self.translated_locale], repositories=[self.repository], ) self.main_db_resource = ResourceFactory.create(project=self.db_project, path="main.lang", format="lang") self.other_db_resource = ResourceFactory.create( project=self.db_project, path="other.lang", format="lang") self.missing_db_resource = ResourceFactory.create( project=self.db_project, path="missing.lang", format="lang") self.main_db_entity = EntityFactory.create( resource=self.main_db_resource, string="Source String", key="Source String", obsolete=False, ) self.other_db_entity = EntityFactory.create( resource=self.other_db_resource, string="Other Source String", key="Other Source String", obsolete=False, ) self.main_db_translation = TranslationFactory.create( entity=self.main_db_entity, plural_form=None, locale=self.translated_locale, string="Translated String", date=aware_datetime(1970, 1, 1), approved=True, extra={"tags": []}, ) # Load paths from the fake locale directory. checkout_path_patch = patch.object( Project, "checkout_path", new_callable=PropertyMock, return_value=FAKE_CHECKOUT_PATH, ) checkout_path_patch.start() self.addCleanup(checkout_path_patch.stop) vcs_changed_files = { self.main_db_resource.path: [self.translated_locale], self.other_db_resource.path: [self.translated_locale], self.missing_db_resource.path: [self.translated_locale], } changed_files_patch = patch.object( VCSProject, "changed_files", new_callable=PropertyMock, return_value=vcs_changed_files, ) changed_files_patch.start() self.addCleanup(changed_files_patch.stop) source_repository = patch.object( Project, "source_repository", new_callable=PropertyMock, return_value=self.db_project.repositories.all()[0], ) source_repository.start() self.addCleanup(source_repository.stop) self.vcs_project = VCSProject(self.db_project) self.main_vcs_resource = self.vcs_project.resources[ self.main_db_resource.path] self.other_vcs_resource = self.vcs_project.resources[ self.other_db_resource.path] self.missing_vcs_resource = self.vcs_project.resources[ self.missing_db_resource.path] self.main_vcs_entity = self.main_vcs_resource.entities["Source String"] self.main_vcs_translation = self.main_vcs_entity.translations[ "translated-locale"] # Mock VCSResource.save() for each resource to avoid altering # the filesystem. resource_save_patch = patch.object(VCSResource, "save") resource_save_patch.start() self.addCleanup(resource_save_patch.stop) self.changeset = ChangeSet( self.db_project, self.vcs_project, aware_datetime(1970, 1, 1), self.translated_locale, )
def sync_project_repo(self, project_pk, repo_pk, project_sync_log_pk, now, no_pull=False, no_commit=False): db_project = get_or_fail(Project, pk=project_pk, message='Could not sync project with pk={0}, not found.'.format(project_pk)) repo = get_or_fail(Repository, pk=repo_pk, message='Could not sync repo with pk={0}, not found.'.format(project_pk)) project_sync_log = get_or_fail(ProjectSyncLog, pk=project_sync_log_pk, message=('Could not sync project {0}, log with pk={1} not found.' .format(db_project.slug, project_sync_log_pk))) repo_sync_log = RepositorySyncLog.objects.create( project_sync_log=project_sync_log, repository=repo, start_time=timezone.now() ) # Pull VCS changes in case we're on a different worker than the one # sync started on. if not no_pull: pull_changes(db_project) if len(repo.locales) < 1: log.warning('Could not sync repo `{0}`, no locales found within.' .format(repo.url)) repo_sync_log.end_time = timezone.now() repo_sync_log.save(update_fields=['end_time']) return vcs_project = VCSProject(db_project, locales=repo.locales) for locale in repo.locales: try: with transaction.atomic(): changeset = ChangeSet(db_project, vcs_project, now) update_translations(db_project, vcs_project, locale, changeset) changeset.execute() update_project_stats(db_project, vcs_project, changeset, locale) # Clear out the "has_changed" markers now that we've finished # syncing. (ChangedEntityLocale.objects .filter(entity__resource__project=db_project, locale=locale, when__lte=now) .delete()) db_project.has_changed = False db_project.save(update_fields=['has_changed']) # Clean up any duplicate approvals at the end of sync right # before we commit the transaction to avoid race conditions. with connection.cursor() as cursor: cursor.execute(""" UPDATE base_translation AS b SET approved = FALSE, approved_date = NULL WHERE id IN (SELECT trans.id FROM base_translation AS trans LEFT JOIN base_entity AS ent ON ent.id = trans.entity_id LEFT JOIN base_resource AS res ON res.id = ent.resource_id WHERE locale_id = %(locale_id)s AND res.project_id = %(project_id)s) AND approved_date != (SELECT max(approved_date) FROM base_translation WHERE entity_id = b.entity_id AND locale_id = b.locale_id AND (plural_form = b.plural_form OR plural_form IS NULL)); """, { 'locale_id': locale.id, 'project_id': db_project.id }) # Perform the commit last so that, if it succeeds, there is # nothing after it to fail. if not no_commit and locale in changeset.locales_to_commit: commit_changes(db_project, vcs_project, changeset, locale) except CommitToRepositoryException as err: # Transaction aborted, log and move on to the next locale. log.warning( 'Failed to sync locale {locale} for project {project} due to ' 'commit error: {error}'.format( locale=locale.code, project=db_project.slug, error=err, ) ) repo_sync_log.end_time = timezone.now() repo_sync_log.save() log.info('Synced translations for project {0} in locales {1}.'.format( db_project.slug, ','.join(locale.code for locale in repo.locales) ))
def sync_translations(self, project_pk, repo_pk, project_sync_log_pk, now, project_changes=None, obsolete_vcs_resources=None, new_paths=None, locale=None, no_pull=False, no_commit=False, full_scan=False): db_project = get_or_fail(Project, pk=project_pk, message='Could not sync project with pk={0}, not found.'.format(project_pk)) repo = get_or_fail(Repository, pk=repo_pk, message='Could not sync repo with pk={0}, not found.'.format(repo_pk)) project_sync_log = get_or_fail(ProjectSyncLog, pk=project_sync_log_pk, message=('Could not sync project {0}, log with pk={1} not found.' .format(db_project.slug, project_sync_log_pk))) log.info('Syncing translations for repo: {}'.format(repo.url)) repo_sync_log = RepositorySyncLog.objects.create( project_sync_log=project_sync_log, repository=repo, start_time=timezone.now() ) if locale: locales = [locale] else: locales = repo.locales if not locales: log.info('Skipping repo `{0}` for project {1}, no locales to sync found within.' .format(repo.url, db_project.slug)) repo_sync_log.end() return # Pull VCS changes in case we're on a different worker than the one # sync started on. if not no_pull: repos_changed = pull_changes(db_project) resources_changed = [] obsolete_vcs_entities = [] if project_changes: resources_changed = ( project_changes['update_db'] + project_changes['obsolete_db'] + project_changes['create_db'] ) obsolete_vcs_entities = project_changes['obsolete_db'] # If none of the repos has changed since the last sync and there are # no Pontoon-side changes for this project, quit early. if not full_scan and not db_project.needs_sync and not repos_changed and not (resources_changed or obsolete_vcs_resources): log.info('Skipping project {0}, no changes detected.'.format(db_project.slug)) repo_sync_log.end() return obsolete_entities_paths = Resource.objects.obsolete_entities_paths(obsolete_vcs_entities) if obsolete_vcs_entities else None vcs_project = VCSProject( db_project, locales=locales, obsolete_entities_paths=obsolete_entities_paths, new_paths=new_paths, full_scan=full_scan ) failed_locales = set() for locale in locales: try: with transaction.atomic(): # Sets VCSProject.synced_locales, needed to skip early if not vcs_project.synced_locales: vcs_project.resources if not obsolete_vcs_resources: # Skip all locales if none of the them has anything to sync if len(vcs_project.synced_locales) == 0: if resources_changed: for l in locales: update_translated_resources(db_project, vcs_project, l) update_locale_project_locale_stats(l, db_project) db_project.aggregate_stats() log.info('Skipping repo `{0}` for project {1}, none of the locales has anything to sync.' .format(repo.url, db_project.slug)) repo.set_last_synced_revisions() repo_sync_log.end() return # Skip locales that have nothing to sync if vcs_project.synced_locales and locale not in vcs_project.synced_locales: if resources_changed: update_translated_resources(db_project, vcs_project, locale) update_locale_project_locale_stats(locale, db_project) log.debug('Skipping locale `{0}` for project {1}, no changes detected.' .format(locale.code, db_project.slug)) continue changeset = ChangeSet(db_project, vcs_project, now, obsolete_vcs_entities, obsolete_vcs_resources, locale) update_translations(db_project, vcs_project, locale, changeset) changeset.execute() update_translated_resources(db_project, vcs_project, locale) update_locale_project_locale_stats(locale, db_project) # Clear out the "has_changed" markers now that we've finished # syncing. (ChangedEntityLocale.objects .filter(entity__resource__project=db_project, locale=locale, when__lte=now) .delete()) # Clean up any duplicate approvals at the end of sync right # before we commit the transaction to avoid race conditions. with connection.cursor() as cursor: cursor.execute(""" UPDATE base_translation AS b SET approved = FALSE, approved_date = NULL WHERE id IN (SELECT trans.id FROM base_translation AS trans LEFT JOIN base_entity AS ent ON ent.id = trans.entity_id LEFT JOIN base_resource AS res ON res.id = ent.resource_id WHERE locale_id = %(locale_id)s AND res.project_id = %(project_id)s) AND approved_date != (SELECT max(approved_date) FROM base_translation WHERE entity_id = b.entity_id AND locale_id = b.locale_id AND (plural_form = b.plural_form OR plural_form IS NULL)); """, { 'locale_id': locale.id, 'project_id': db_project.id }) # Perform the commit last so that, if it succeeds, there is # nothing after it to fail. if not no_commit and locale in changeset.locales_to_commit: commit_changes(db_project, vcs_project, changeset, locale) log.info( 'Synced locale {locale} for project {project}.'.format( locale=locale.code, project=db_project.slug, ) ) except CommitToRepositoryException as err: # Transaction aborted, log and move on to the next locale. log.warning( 'Failed to sync locale {locale} for project {project} due to ' 'commit error: {error}'.format( locale=locale.code, project=db_project.slug, error=err, ) ) failed_locales.add(locale) with transaction.atomic(): db_project.aggregate_stats() synced_locales = [locale.code for locale in (vcs_project.synced_locales - failed_locales)] if synced_locales: log.info('Synced translations for project {0} in locales {1}.'.format( db_project.slug, ','.join(synced_locales) )) else: log.info('Failed to sync translations for project {0} due to commit error.'.format( db_project.slug )) repo.set_last_synced_revisions(exclude=failed_locales) repo_sync_log.end()
def handle_upload_content(slug, code, part, f, user): """ Update translations in the database from uploaded file. :arg str slug: Project slug. :arg str code: Locale code. :arg str part: Resource path or Subpage name. :arg UploadedFile f: UploadedFile instance. :arg User user: User uploading the file. """ # Avoid circular import; someday we should refactor to avoid. from pontoon.sync import formats from pontoon.sync.changeset import ChangeSet from pontoon.sync.vcs.models import VCSProject from pontoon.base.models import ( ChangedEntityLocale, Entity, Locale, Project, Resource, TranslatedResource, Translation, ) relative_path = _get_relative_path_from_part(slug, part) project = get_object_or_404(Project, slug=slug) locale = get_object_or_404(Locale, code=code) resource = get_object_or_404(Resource, project__slug=slug, path=relative_path) # Store uploaded file to a temporary file and parse it extension = os.path.splitext(f.name)[1] with tempfile.NamedTemporaryFile( prefix='strings' if extension == '.xml' else '', suffix=extension, ) as temp: for chunk in f.chunks(): temp.write(chunk) temp.flush() resource_file = formats.parse(temp.name) # Update database objects from file changeset = ChangeSet( project, VCSProject(project, locales=[locale]), timezone.now() ) entities_qs = Entity.objects.filter( resource__project=project, resource__path=relative_path, obsolete=False ).prefetch_related( Prefetch( 'translation_set', queryset=Translation.objects.filter(locale=locale), to_attr='db_translations' ) ).prefetch_related( Prefetch( 'translation_set', queryset=Translation.objects.filter(locale=locale, approved_date__lte=timezone.now()), to_attr='db_translations_approved_before_sync' ) ) entities_dict = {entity.key: entity for entity in entities_qs} for vcs_translation in resource_file.translations: key = vcs_translation.key if key in entities_dict: entity = entities_dict[key] changeset.update_entity_translations_from_vcs( entity, locale.code, vcs_translation, user, entity.db_translations, entity.db_translations_approved_before_sync ) changeset.bulk_create_translations() changeset.bulk_update_translations() if changeset.changed_translations: # Update 'active' status of all changed translations and their siblings, # i.e. translations of the same entity to the same locale. changed_pks = {t.pk for t in changeset.changed_translations} ( Entity.objects .filter(translation__pk__in=changed_pks) .reset_active_translations(locale=locale) ) # Run checks and create TM entries for translations that pass them valid_translations = changeset.bulk_check_translations() changeset.bulk_create_translation_memory_entries(valid_translations) TranslatedResource.objects.get(resource=resource, locale=locale).calculate_stats() # Mark translations as changed changed_entities = {} existing = ChangedEntityLocale.objects.values_list('entity', 'locale').distinct() for t in changeset.changed_translations: key = (t.entity.pk, t.locale.pk) # Remove duplicate changes to prevent unique constraint violation if key not in existing: changed_entities[key] = ChangedEntityLocale(entity=t.entity, locale=t.locale) ChangedEntityLocale.objects.bulk_create(changed_entities.values()) # Update latest translation if changeset.translations_to_create: changeset.translations_to_create[-1].update_latest_translation()
def sync_translations( self, project_pk, project_sync_log_pk, now, added_paths=None, removed_paths=None, changed_paths=None, locale=None, no_pull=False, no_commit=False, full_scan=False, ): db_project = get_or_fail( Project, pk=project_pk, message='Could not sync project with pk={0}, not found.'.format(project_pk) ) repos = db_project.translation_repositories() repo_pk = repos[0].pk repo = get_or_fail( Repository, pk=repo_pk, message='Could not sync repo with pk={0}, not found.'.format(repo_pk) ) project_sync_log = get_or_fail( ProjectSyncLog, pk=project_sync_log_pk, message=( 'Could not sync project {0}, log with pk={1} not found.' .format(db_project.slug, project_sync_log_pk) ) ) log.info('Syncing translations for project: {}'.format(db_project.slug)) repo_sync_log = RepositorySyncLog.objects.create( project_sync_log=project_sync_log, repository=repo, start_time=timezone.now() ) if locale: locales = db_project.locales.filter(pk=locale.pk) else: locales = db_project.locales.all() if not locales: log.info('Skipping syncing translations for project {0}, no locales to sync found within.' .format(db_project.slug)) repo_sync_log.end() return # If project repositories have API access, we can retrieve latest commit hashes and detect # changed locales before the expensive VCS pull/clone operations. When performing full scan, # we still need to sync all locales. if not full_scan: locales = get_changed_locales(db_project, locales, now) readonly_locales = db_project.locales.filter(project_locale__readonly=True) added_and_changed_resources = db_project.resources.filter( path__in=list(added_paths or []) + list(changed_paths or []) ).distinct() # We should also sync files for which source file change - but only for read-only locales. # See bug 1372151 for more details. if added_and_changed_resources: changed_locales_pks = [l.pk for l in locales] readonly_locales_pks = [l.pk for l in readonly_locales] locales = db_project.locales.filter( pk__in=changed_locales_pks + readonly_locales_pks ) # Pull VCS changes in case we're on a different worker than the one # sync started on. if not no_pull: log.info('Pulling changes for project {0} started.'.format(db_project.slug)) repos_changed, repo_locales = pull_changes(db_project, locales) repos = repos.filter(pk__in=repo_locales.keys()) log.info('Pulling changes for project {0} complete.'.format(db_project.slug)) # If none of the repos has changed since the last sync and there are # no Pontoon-side changes for this project, quit early. if ( not full_scan and not db_project.needs_sync and not repos_changed and not (added_paths or removed_paths or changed_paths) ): log.info('Skipping project {0}, no changes detected.'.format(db_project.slug)) repo_sync_log.end() return vcs_project = VCSProject( db_project, now, locales=locales, repo_locales=repo_locales, added_paths=added_paths, changed_paths=changed_paths, full_scan=full_scan ) synced_locales = set() failed_locales = set() for locale in locales: try: with transaction.atomic(): # Sets VCSProject.synced_locales, needed to skip early if not vcs_project.synced_locales: vcs_project.resources # Skip all locales if none of the them has anything to sync if len(vcs_project.synced_locales) == 0: break # Skip locales that have nothing to sync if vcs_project.synced_locales and locale not in vcs_project.synced_locales: continue changeset = ChangeSet(db_project, vcs_project, now, locale) update_translations(db_project, vcs_project, locale, changeset) changeset.execute() update_translated_resources(db_project, vcs_project, locale) update_locale_project_locale_stats(locale, db_project) # Clear out the "has_changed" markers now that we've finished # syncing. (ChangedEntityLocale.objects .filter(entity__resource__project=db_project, locale=locale, when__lte=now) .delete()) # Perform the commit last so that, if it succeeds, there is # nothing after it to fail. if ( not no_commit and locale in changeset.locales_to_commit and locale not in readonly_locales ): commit_changes(db_project, vcs_project, changeset, locale) log.info( 'Synced locale {locale} for project {project}.'.format( locale=locale.code, project=db_project.slug, ) ) synced_locales.add(locale.code) except CommitToRepositoryException as err: # Transaction aborted, log and move on to the next locale. log.warning( 'Failed to sync locale {locale} for project {project} due to ' 'commit error: {error}'.format( locale=locale.code, project=db_project.slug, error=err, ) ) failed_locales.add(locale.code) # If sources have changed, update stats for all locales. if added_paths or removed_paths or changed_paths: for locale in db_project.locales.all(): # Already synced. if locale.code in synced_locales: continue # We have files: update all translated resources. if locale in locales: update_translated_resources(db_project, vcs_project, locale) # We don't have files: we can still update asymmetric translated resources. else: update_translated_resources_no_files( db_project, locale, added_and_changed_resources, ) update_locale_project_locale_stats(locale, db_project) synced_locales.add(locale.code) log.info( 'Synced source changes for locale {locale} for project {project}.'.format( locale=locale.code, project=db_project.slug, ) ) db_project.aggregate_stats() if synced_locales: log.info('Synced translations for project {0} in locales {1}.'.format( db_project.slug, ','.join(synced_locales) )) elif failed_locales: log.info('Failed to sync translations for project {0} due to commit error.'.format( db_project.slug )) else: log.info( 'Skipping syncing translations for project {0}, none of the locales ' 'has anything to sync.'.format(db_project.slug) ) for r in repos: r.set_last_synced_revisions( locales=repo_locales[r.pk].exclude(code__in=failed_locales) ) repo_sync_log.end()
def sync_project_repo(self, project_pk, repo_pk, project_sync_log_pk, now, obsolete_vcs_entities=None, obsolete_vcs_resources=None, locale=None, no_pull=False, no_commit=False, full_scan=False): db_project = get_or_fail(Project, pk=project_pk, message='Could not sync project with pk={0}, not found.'.format(project_pk)) repo = get_or_fail(Repository, pk=repo_pk, message='Could not sync repo with pk={0}, not found.'.format(repo_pk)) project_sync_log = get_or_fail(ProjectSyncLog, pk=project_sync_log_pk, message=('Could not sync project {0}, log with pk={1} not found.' .format(db_project.slug, project_sync_log_pk))) log.info('Syncing repo: {}'.format(repo.url)) repo_sync_log = RepositorySyncLog.objects.create( project_sync_log=project_sync_log, repository=repo, start_time=timezone.now() ) # Pull VCS changes in case we're on a different worker than the one # sync started on. if not no_pull: pull_changes(db_project) if locale: locales = [locale] else: locales = repo.locales vcs_project = VCSProject( db_project, locales=locales, obsolete_entities_paths=Resource.objects.obsolete_entities_paths(obsolete_vcs_entities), full_scan=full_scan ) for locale in locales: try: with transaction.atomic(): # Skip locales that have nothing to sync if vcs_project.synced_locales and locale not in vcs_project.synced_locales: continue changeset = ChangeSet(db_project, vcs_project, now, obsolete_vcs_entities, obsolete_vcs_resources) update_translations(db_project, vcs_project, locale, changeset) changeset.execute() update_translated_resources(db_project, vcs_project, changeset, locale) # Skip if none of the locales has anything to sync # VCSProject.synced_locales is set on a first call to # VCSProject.resources, which is set in # pontoon.sync.core.update_translated_resources() if len(vcs_project.synced_locales) == 0: log.info('Skipping repo `{0}` for project {1}, none of the locales has anything to sync.' .format(repo.url, db_project.slug)) end_repo_sync(repo, repo_sync_log) return locale.aggregate_stats() locale.project_locale.get(project=db_project).aggregate_stats() # Clear out the "has_changed" markers now that we've finished # syncing. (ChangedEntityLocale.objects .filter(entity__resource__project=db_project, locale=locale, when__lte=now) .delete()) db_project.has_changed = False db_project.save(update_fields=['has_changed']) # Clean up any duplicate approvals at the end of sync right # before we commit the transaction to avoid race conditions. with connection.cursor() as cursor: cursor.execute(""" UPDATE base_translation AS b SET approved = FALSE, approved_date = NULL WHERE id IN (SELECT trans.id FROM base_translation AS trans LEFT JOIN base_entity AS ent ON ent.id = trans.entity_id LEFT JOIN base_resource AS res ON res.id = ent.resource_id WHERE locale_id = %(locale_id)s AND res.project_id = %(project_id)s) AND approved_date != (SELECT max(approved_date) FROM base_translation WHERE entity_id = b.entity_id AND locale_id = b.locale_id AND (plural_form = b.plural_form OR plural_form IS NULL)); """, { 'locale_id': locale.id, 'project_id': db_project.id }) # Perform the commit last so that, if it succeeds, there is # nothing after it to fail. if not no_commit and locale in changeset.locales_to_commit: commit_changes(db_project, vcs_project, changeset, locale) except CommitToRepositoryException as err: # Transaction aborted, log and move on to the next locale. log.warning( 'Failed to sync locale {locale} for project {project} due to ' 'commit error: {error}'.format( locale=locale.code, project=db_project.slug, error=err, ) ) with transaction.atomic(): db_project.aggregate_stats() log.info('Synced translations for project {0} in locales {1}.'.format( db_project.slug, ','.join(locale.code for locale in vcs_project.synced_locales) )) end_repo_sync(repo, repo_sync_log)
def sync_project(db_project, no_pull=False, no_commit=False): """ Update the database with the current state of resources in version control and write any submitted translations from the database back to version control. """ # Mark "now" at the start of sync to avoid messing with # translations submitted during sync. now = timezone.now() # Pull changes from VCS and update what we know about the files. if not no_pull: repos_changed = pull_changes(db_project) else: repos_changed = True # Assume changed. # If the repos haven't changed since the last sync and there are # no Pontoon-side changes for this project, quit early. if not repos_changed and not db_project.needs_sync: log.info('Skipping project {0}, no changes detected.'.format(db_project.slug)) return vcs_project = VCSProject(db_project) update_resources(db_project, vcs_project) # Collect all entities across VCS and the database and get their # keys so we can match up matching entities. vcs_entities = get_vcs_entities(vcs_project) db_entities = get_db_entities(db_project) entity_keys = set().union(db_entities.keys(), vcs_entities.keys()) changeset = ChangeSet(db_project, vcs_project, now) for key in entity_keys: db_entity = db_entities.get(key, None) vcs_entity = vcs_entities.get(key, None) handle_entity(changeset, db_project, key, db_entity, vcs_entity) # Apply the changeset to the files, commit them, and update stats # entries in the DB. changeset.execute() if not no_commit: commit_changes(db_project, vcs_project, changeset) update_project_stats(db_project, vcs_project, changeset) # Clear out the "has_changed" markers now that we've finished # syncing. (ChangedEntityLocale.objects .filter(entity__resource__project=db_project, when__lte=now) .delete()) db_project.has_changed = False db_project.save() # Clean up any duplicate approvals at the end of sync right # before we commit the transaction to avoid race conditions. with connection.cursor() as cursor: cursor.execute(""" UPDATE base_translation AS b SET approved = FALSE, approved_date = NULL WHERE approved_date != (SELECT max(approved_date) FROM base_translation WHERE entity_id = b.entity_id AND locale_id = b.locale_id AND (plural_form = b.plural_form OR plural_form IS NULL)); """) log.info(u'Synced project {0}'.format(db_project.slug))
def handle_upload_content(slug, code, part, f, user): """ Update translations in the database from uploaded file. :arg str slug: Project slug. :arg str code: Locale code. :arg str part: Resource path or Subpage name. :arg UploadedFile f: UploadedFile instance. :arg User user: User uploading the file. """ # Avoid circular import; someday we should refactor to avoid. from pontoon.sync import formats from pontoon.sync.changeset import ChangeSet from pontoon.sync.vcs.models import VCSProject from pontoon.base.models import ( ChangedEntityLocale, Entity, Locale, Project, Resource, TranslatedResource, Translation, ) relative_path = _get_relative_path_from_part(slug, part) project = get_object_or_404(Project, slug=slug) locale = get_object_or_404(Locale, code=code) resource = get_object_or_404(Resource, project__slug=slug, path=relative_path) # Store uploaded file to a temporary file and parse it extension = os.path.splitext(f.name)[1] with tempfile.NamedTemporaryFile(suffix=extension) as temp: for chunk in f.chunks(): temp.write(chunk) temp.flush() resource_file = formats.parse(temp.name) # Update database objects from file changeset = ChangeSet(project, VCSProject(project, locales=[locale]), timezone.now()) entities_qs = Entity.objects.filter( resource__project=project, resource__path=relative_path, obsolete=False).prefetch_related( Prefetch('translation_set', queryset=Translation.objects.filter(locale=locale), to_attr='db_translations')).prefetch_related( Prefetch( 'translation_set', queryset=Translation.objects.filter( locale=locale, approved_date__lte=timezone.now()), to_attr='db_translations_approved_before_sync')) entities_dict = {entity.key: entity for entity in entities_qs} for vcs_translation in resource_file.translations: key = vcs_translation.key if key in entities_dict: entity = entities_dict[key] changeset.update_entity_translations_from_vcs( entity, locale.code, vcs_translation, user, entity.db_translations, entity.db_translations_approved_before_sync) changeset.bulk_create_translations() changeset.bulk_update_translations() changeset.bulk_create_translaton_memory_entries() TranslatedResource.objects.get(resource=resource, locale=locale).calculate_stats() # Mark translations as changed changed_entities = {} existing = ChangedEntityLocale.objects.values_list('entity', 'locale').distinct() for t in changeset.changed_translations: key = (t.entity.pk, t.locale.pk) # Remove duplicate changes to prevent unique constraint violation if key not in existing: changed_entities[key] = ChangedEntityLocale(entity=t.entity, locale=t.locale) ChangedEntityLocale.objects.bulk_create(changed_entities.values()) # Update latest translation if changeset.translations_to_create: changeset.translations_to_create[-1].update_latest_translation()
def sync_translations( self, project_pk, project_sync_log_pk, now, project_changes=None, obsolete_vcs_resources=None, new_paths=None, locale=None, no_pull=False, no_commit=False, full_scan=False ): db_project = get_or_fail( Project, pk=project_pk, message='Could not sync project with pk={0}, not found.'.format(project_pk) ) repos = db_project.translation_repositories() repo_pk = repos[0].pk repo = get_or_fail( Repository, pk=repo_pk, message='Could not sync repo with pk={0}, not found.'.format(repo_pk) ) project_sync_log = get_or_fail( ProjectSyncLog, pk=project_sync_log_pk, message=( 'Could not sync project {0}, log with pk={1} not found.' .format(db_project.slug, project_sync_log_pk) ) ) log.info('Syncing translations for project: {}'.format(db_project.slug)) repo_sync_log = RepositorySyncLog.objects.create( project_sync_log=project_sync_log, repository=repo, start_time=timezone.now() ) if locale: locales = db_project.locales.filter(pk=locale.pk) else: locales = db_project.locales.all() if not locales: log.info('Skipping syncing translations for project {0}, no locales to sync found within.' .format(db_project.slug)) repo_sync_log.end() return # If project repositories have API access, we can retrieve latest commit hashes and detect # changed locales before the expensive VCS pull/clone operations. When performing full scan, # we still need to sync all locales. if not full_scan: locales = get_changed_locales(db_project, locales, now) # Pull VCS changes in case we're on a different worker than the one # sync started on. if not no_pull: log.info('Pulling changes for project {0} started.'.format(db_project.slug)) repos_changed, repo_locales = pull_changes(db_project, locales) repos = repos.filter(pk__in=repo_locales.keys()) log.info('Pulling changes for project {0} complete.'.format(db_project.slug)) changed_resources = [] obsolete_vcs_entities = [] if project_changes: updated_entity_pks = [] for locale_code, db_entity, vcs_entity in project_changes['update_db']: updated_entity_pks.append(db_entity.pk) obsolete_entity_pks = project_changes['obsolete_db'] changed_resources = db_project.resources.filter( Q(entities__date_created=now) | Q(entities__pk__in=updated_entity_pks + obsolete_entity_pks) ).distinct() obsolete_vcs_entities = project_changes['obsolete_db'] # If none of the repos has changed since the last sync and there are # no Pontoon-side changes for this project, quit early. if ( not full_scan and not db_project.needs_sync and not repos_changed and not (changed_resources or obsolete_vcs_resources) ): log.info('Skipping project {0}, no changes detected.'.format(db_project.slug)) repo_sync_log.end() return obsolete_entities_paths = ( Resource.objects.obsolete_entities_paths(obsolete_vcs_entities) if obsolete_vcs_entities else None ) vcs_project = VCSProject( db_project, now, locales=locales, repo_locales=repo_locales, obsolete_entities_paths=obsolete_entities_paths, new_paths=new_paths, full_scan=full_scan ) synced_locales = set() failed_locales = set() for locale in locales: try: with transaction.atomic(): # Sets VCSProject.synced_locales, needed to skip early if not vcs_project.synced_locales: vcs_project.resources # Skip all locales if none of the them has anything to sync if len(vcs_project.synced_locales) == 0: break # Skip locales that have nothing to sync if vcs_project.synced_locales and locale not in vcs_project.synced_locales: continue changeset = ChangeSet(db_project, vcs_project, now, locale) update_translations(db_project, vcs_project, locale, changeset) changeset.execute() update_translated_resources(db_project, vcs_project, locale) update_locale_project_locale_stats(locale, db_project) # Clear out the "has_changed" markers now that we've finished # syncing. (ChangedEntityLocale.objects .filter(entity__resource__project=db_project, locale=locale, when__lte=now) .delete()) # Clean up any duplicate approvals at the end of sync right # before we commit the transaction to avoid race conditions. with connection.cursor() as cursor: cursor.execute(""" UPDATE base_translation AS b SET approved = FALSE, approved_date = NULL WHERE id IN (SELECT trans.id FROM base_translation AS trans LEFT JOIN base_entity AS ent ON ent.id = trans.entity_id LEFT JOIN base_resource AS res ON res.id = ent.resource_id WHERE locale_id = %(locale_id)s AND res.project_id = %(project_id)s) AND approved_date != (SELECT max(approved_date) FROM base_translation WHERE entity_id = b.entity_id AND locale_id = b.locale_id AND (plural_form = b.plural_form OR plural_form IS NULL)); """, { 'locale_id': locale.id, 'project_id': db_project.id }) # Perform the commit last so that, if it succeeds, there is # nothing after it to fail. if not no_commit and locale in changeset.locales_to_commit: commit_changes(db_project, vcs_project, changeset, locale) log.info( 'Synced locale {locale} for project {project}.'.format( locale=locale.code, project=db_project.slug, ) ) synced_locales.add(locale.code) except CommitToRepositoryException as err: # Transaction aborted, log and move on to the next locale. log.warning( 'Failed to sync locale {locale} for project {project} due to ' 'commit error: {error}'.format( locale=locale.code, project=db_project.slug, error=err, ) ) failed_locales.add(locale.code) # If sources have changed, update stats for all locales. if changed_resources or obsolete_vcs_resources: for locale in db_project.locales.all(): # Already synced. if locale.code in synced_locales: continue # We have files: update all translated resources. if locale in locales: update_translated_resources(db_project, vcs_project, locale) # We don't have files: we can still update asymmetric translated resources. else: update_translated_resources_no_files(db_project, locale, changed_resources) update_locale_project_locale_stats(locale, db_project) synced_locales.add(locale.code) log.info( 'Synced source changes for locale {locale} for project {project}.'.format( locale=locale.code, project=db_project.slug, ) ) db_project.aggregate_stats() if synced_locales: log.info('Synced translations for project {0} in locales {1}.'.format( db_project.slug, ','.join(synced_locales) )) elif failed_locales: log.info('Failed to sync translations for project {0} due to commit error.'.format( db_project.slug )) else: log.info( 'Skipping syncing translations for project {0}, none of the locales ' 'has anything to sync.'.format(db_project.slug) ) for r in repos: r.set_last_synced_revisions( locales=repo_locales[r.pk].exclude(code__in=failed_locales) ) repo_sync_log.end()