Ejemplo n.º 1
0
def sync_project(db_project, now):
    # Only load source resources for updating entities.
    vcs_project = VCSProject(db_project, locales=[])
    with transaction.atomic():
        update_resources(db_project, vcs_project)
        changeset = ChangeSet(db_project, vcs_project, now)
        update_entities(db_project, vcs_project, changeset)
        changeset.execute()
Ejemplo n.º 2
0
def sync_project(db_project, now):
    # Only load source resources for updating entities.
    vcs_project = VCSProject(db_project, locales=[])
    with transaction.atomic():
        update_resources(db_project, vcs_project)
        changeset = ChangeSet(db_project, vcs_project, now)
        update_entities(db_project, vcs_project, changeset)
        changeset.execute()
Ejemplo n.º 3
0
def sync_project(db_project, now, full_scan=False):
    vcs_project = VCSProject(db_project, locales=[], full_scan=full_scan)

    with transaction.atomic():
        removed_paths = update_resources(db_project, vcs_project)
        changeset = ChangeSet(db_project, vcs_project, now)
        update_entities(db_project, vcs_project, changeset)
        changeset.execute()

    return changeset.changes['obsolete_db'], removed_paths
Ejemplo n.º 4
0
def sync_project(db_project, now):
    # Only load source resources for updating entities.
    vcs_project = VCSProject(db_project, locales=[])
    with transaction.atomic():
        removed_paths = update_resources(db_project, vcs_project)
        changeset = ChangeSet(db_project, vcs_project, now)
        update_entities(db_project, vcs_project, changeset)
        changeset.execute()

    return changeset.changes['obsolete_db'], removed_paths
Ejemplo n.º 5
0
def sync_project(db_project, now, full_scan=False):
    vcs_project = VCSProject(db_project, locales=[], full_scan=full_scan)

    with transaction.atomic():
        removed_paths = update_resources(db_project, vcs_project)
        changeset = ChangeSet(db_project, vcs_project, now)
        update_entities(db_project, vcs_project, changeset)
        changeset.execute()

    return changeset.changes['obsolete_db'], removed_paths
Ejemplo n.º 6
0
def update_originals(db_project, now, full_scan=False):
    vcs_project = VCSProject(db_project, locales=[], full_scan=full_scan)

    with transaction.atomic():
        added_paths, removed_paths, changed_paths = update_resources(db_project, vcs_project)
        changeset = ChangeSet(db_project, vcs_project, now)
        update_entities(db_project, vcs_project, changeset)
        changeset.execute()

    return added_paths, removed_paths, changed_paths
Ejemplo n.º 7
0
    def setUp(self):
        self.now = aware_datetime(1970, 1, 1)

        timezone_patch = patch('pontoon.sync.tasks.timezone')
        self.mock_timezone = timezone_patch.start()
        self.addCleanup(timezone_patch.stop)
        self.mock_timezone.now.return_value = self.now

        self.translated_locale = LocaleFactory.create(code='translated-locale')
        self.inactive_locale = LocaleFactory.create(code='inactive-locale')
        self.repository = RepositoryFactory()

        self.db_project = ProjectFactory.create(
            name='db-project',
            locales=[self.translated_locale],
            repositories=[self.repository])
        self.main_db_resource = ResourceFactory.create(project=self.db_project,
                                                       path='main.lang',
                                                       format='lang')
        self.other_db_resource = ResourceFactory.create(
            project=self.db_project, path='other.lang', format='lang')
        self.missing_db_resource = ResourceFactory.create(
            project=self.db_project, path='missing.lang', format='lang')
        self.main_db_entity = EntityFactory.create(
            resource=self.main_db_resource,
            string='Source String',
            key='Source String',
            obsolete=False)
        self.other_db_entity = EntityFactory.create(
            resource=self.other_db_resource,
            string='Other Source String',
            key='Other Source String',
            obsolete=False)
        self.main_db_translation = TranslationFactory.create(
            entity=self.main_db_entity,
            plural_form=None,
            locale=self.translated_locale,
            string='Translated String',
            date=aware_datetime(1970, 1, 1),
            approved=True,
            extra={'tags': []})

        # Load paths from the fake locale directory.
        checkout_path_patch = patch.object(Project,
                                           'checkout_path',
                                           new_callable=PropertyMock,
                                           return_value=FAKE_CHECKOUT_PATH)
        checkout_path_patch.start()
        self.addCleanup(checkout_path_patch.stop)

        self.vcs_project = VCSProject(self.db_project)
        self.main_vcs_resource = self.vcs_project.resources[
            self.main_db_resource.path]
        self.other_vcs_resource = self.vcs_project.resources[
            self.other_db_resource.path]
        self.missing_vcs_resource = self.vcs_project.resources[
            self.missing_db_resource.path]
        self.main_vcs_entity = self.main_vcs_resource.entities['Source String']
        self.main_vcs_translation = self.main_vcs_entity.translations[
            'translated-locale']

        # Mock VCSResource.save() for each resource to avoid altering
        # the filesystem.
        resource_save_patch = patch.object(VCSResource, 'save')
        resource_save_patch.start()
        self.addCleanup(resource_save_patch.stop)

        self.changeset = ChangeSet(self.db_project, self.vcs_project,
                                   aware_datetime(1970, 1, 1))
Ejemplo n.º 8
0
def handle_upload_content(slug, code, part, f, user):
    """
    Update translations in the database from uploaded file.

    :param str slug: Project slug.
    :param str code: Locale code.
    :param str part: Resource path or Subpage name.
    :param UploadedFile f: UploadedFile instance.
    :param User user: User uploading the file.
    """
    # Avoid circular import; someday we should refactor to avoid.
    from pontoon.sync import formats
    from pontoon.sync.changeset import ChangeSet
    from pontoon.sync.vcs.models import VCSProject
    from pontoon.base.models import (
        ChangedEntityLocale,
        Entity,
        Locale,
        Project,
        Resource,
        TranslatedResource,
        Translation,
    )

    relative_path = _get_relative_path_from_part(slug, part)
    project = get_object_or_404(Project, slug=slug)
    locale = get_object_or_404(Locale, code__iexact=code)
    resource = get_object_or_404(Resource, project__slug=slug, path=relative_path)

    # Store uploaded file to a temporary file and parse it
    extension = os.path.splitext(f.name)[1]
    with tempfile.NamedTemporaryFile(suffix=extension) as temp:
        for chunk in f.chunks():
            temp.write(chunk)
        temp.flush()
        resource_file = formats.parse(temp.name)

    # Update database objects from file
    changeset = ChangeSet(
        project,
        VCSProject(project, locales=[locale]),
        timezone.now()
    )
    entities_qs = Entity.objects.filter(
        resource__project=project,
        resource__path=relative_path,
        obsolete=False
    ).prefetch_related(
        Prefetch(
            'translation_set',
            queryset=Translation.objects.filter(locale=locale),
            to_attr='db_translations'
        )
    ).prefetch_related(
        Prefetch(
            'translation_set',
            queryset=Translation.objects.filter(locale=locale, approved_date__lte=timezone.now()),
            to_attr='old_translations'
        )
    )
    entities_dict = {entity.key: entity for entity in entities_qs}

    for vcs_translation in resource_file.translations:
        key = vcs_translation.key
        if key in entities_dict:
            entity = entities_dict[key]
            changeset.update_entity_translations_from_vcs(
                entity, locale.code, vcs_translation, user,
                entity.db_translations, entity.old_translations
            )

    changeset.bulk_create_translations()
    changeset.bulk_update_translations()
    TranslatedResource.objects.get(resource=resource, locale=locale).calculate_stats()

    # Mark translations as changed
    changed_entities = {}
    existing = ChangedEntityLocale.objects.values_list('entity', 'locale').distinct()
    for t in changeset.translations_to_create + changeset.translations_to_update:
        key = (t.entity.pk, t.locale.pk)
        # Remove duplicate changes to prevent unique constraint violation
        if not key in existing:
            changed_entities[key] = ChangedEntityLocale(entity=t.entity, locale=t.locale)

    ChangedEntityLocale.objects.bulk_create(changed_entities.values())
Ejemplo n.º 9
0
def sync_translations(
    self,
    project_pk,
    project_sync_log_pk,
    now,
    added_paths=None,
    removed_paths=None,
    changed_paths=None,
    new_entities=None,
    locale=None,
    no_pull=False,
    no_commit=False,
    full_scan=False,
):
    db_project = get_or_fail(
        Project,
        pk=project_pk,
        message="Could not sync project with pk={0}, not found.".format(project_pk),
    )

    repos = db_project.translation_repositories()
    repo_pk = repos[0].pk
    repo = get_or_fail(
        Repository,
        pk=repo_pk,
        message="Could not sync repo with pk={0}, not found.".format(repo_pk),
    )

    project_sync_log = get_or_fail(
        ProjectSyncLog,
        pk=project_sync_log_pk,
        message=(
            "Could not sync project {0}, log with pk={1} not found.".format(
                db_project.slug, project_sync_log_pk
            )
        ),
    )

    log.info("Syncing translations for project: {}".format(db_project.slug))

    repo_sync_log = RepositorySyncLog.objects.create(
        project_sync_log=project_sync_log, repository=repo, start_time=timezone.now()
    )

    if locale:
        locales = db_project.locales.filter(pk=locale.pk)
    else:
        locales = db_project.locales.all()

    if not locales:
        log.info(
            "Skipping syncing translations for project {0}, no locales to sync "
            "found within.".format(db_project.slug)
        )
        repo_sync_log.end()
        return

    # If project repositories have API access, we can retrieve latest commit hashes and detect
    # changed locales before the expensive VCS pull/clone operations. When performing full scan,
    # we still need to sync all locales.
    if not full_scan:
        locales = get_changed_locales(db_project, locales, now)

    readonly_locales = db_project.locales.filter(project_locale__readonly=True)
    added_and_changed_resources = db_project.resources.filter(
        path__in=list(added_paths or []) + list(changed_paths or [])
    ).distinct()

    # We should also sync files for which source file change - but only for read-only locales.
    # See bug 1372151 for more details.
    if added_and_changed_resources:
        changed_locales_pks = [l.pk for l in locales]
        readonly_locales_pks = [l.pk for l in readonly_locales]
        locales = db_project.locales.filter(
            pk__in=changed_locales_pks + readonly_locales_pks
        )

    # Pull VCS changes in case we're on a different worker than the one
    # sync started on.
    if not no_pull:
        log.info("Pulling changes for project {0} started.".format(db_project.slug))
        repos_changed, repo_locales = pull_changes(db_project, locales)
        repos = repos.filter(pk__in=repo_locales.keys())
        log.info("Pulling changes for project {0} complete.".format(db_project.slug))

    # If none of the repos has changed since the last sync and there are
    # no Pontoon-side changes for this project, quit early.
    if (
        not full_scan
        and not db_project.needs_sync
        and not repos_changed
        and not (added_paths or removed_paths or changed_paths)
    ):
        log.info("Skipping project {0}, no changes detected.".format(db_project.slug))
        repo_sync_log.end()
        return

    vcs_project = VCSProject(
        db_project,
        now,
        locales=locales,
        repo_locales=repo_locales,
        added_paths=added_paths,
        changed_paths=changed_paths,
        full_scan=full_scan,
    )

    synced_locales = set()
    failed_locales = set()

    # Store newly added locales and locales with newly added resources
    new_locales = []

    for locale in locales:
        try:
            with transaction.atomic():
                # Sets VCSProject.synced_locales, needed to skip early
                if not vcs_project.synced_locales:
                    vcs_project.resources

                # Skip all locales if none of the them has anything to sync
                if len(vcs_project.synced_locales) == 0:
                    break

                # Skip locales that have nothing to sync
                if (
                    vcs_project.synced_locales
                    and locale not in vcs_project.synced_locales
                ):
                    continue

                changeset = ChangeSet(db_project, vcs_project, now, locale)
                update_translations(db_project, vcs_project, locale, changeset)
                changeset.execute()

                created = update_translated_resources(db_project, vcs_project, locale)
                if created:
                    new_locales.append(locale.pk)
                update_locale_project_locale_stats(locale, db_project)

                # Clear out the "has_changed" markers now that we've finished
                # syncing.
                (
                    ChangedEntityLocale.objects.filter(
                        entity__resource__project=db_project,
                        locale=locale,
                        when__lte=now,
                    ).delete()
                )

                # Perform the commit last so that, if it succeeds, there is
                # nothing after it to fail.
                if (
                    not no_commit
                    and locale in changeset.locales_to_commit
                    and locale not in readonly_locales
                ):
                    commit_changes(db_project, vcs_project, changeset, locale)

                log.info(
                    "Synced locale {locale} for project {project}.".format(
                        locale=locale.code, project=db_project.slug,
                    )
                )

                synced_locales.add(locale.code)

        except CommitToRepositoryException as err:
            # Transaction aborted, log and move on to the next locale.
            log.warning(
                "Failed to sync locale {locale} for project {project} due to "
                "commit error: {error}".format(
                    locale=locale.code, project=db_project.slug, error=err,
                )
            )

            failed_locales.add(locale.code)

    # If sources have changed, update stats for all locales.
    if added_paths or removed_paths or changed_paths:
        for locale in db_project.locales.all():
            # Already synced.
            if locale.code in synced_locales:
                continue

            # We have files: update all translated resources.
            if locale in locales:
                created = update_translated_resources(db_project, vcs_project, locale)
                if created:
                    new_locales.append[locale.pk]

            # We don't have files: we can still update asymmetric translated resources.
            else:
                update_translated_resources_no_files(
                    db_project, locale, added_and_changed_resources,
                )

            update_locale_project_locale_stats(locale, db_project)
            synced_locales.add(locale.code)

            log.info(
                "Synced source changes for locale {locale} for project {project}.".format(
                    locale=locale.code, project=db_project.slug,
                )
            )

        db_project.aggregate_stats()

    if synced_locales:
        log.info(
            "Synced translations for project {0} in locales {1}.".format(
                db_project.slug, ",".join(synced_locales)
            )
        )
    elif failed_locales:
        log.info(
            "Failed to sync translations for project {0} due to commit error.".format(
                db_project.slug
            )
        )
    else:
        log.info(
            "Skipping syncing translations for project {0}, none of the locales "
            "has anything to sync.".format(db_project.slug)
        )

    for r in repos:
        r.set_last_synced_revisions(
            locales=repo_locales[r.pk].exclude(code__in=failed_locales)
        )
    repo_sync_log.end()

    if db_project.pretranslation_enabled:
        # Pretranslate all entities for newly added locales
        # and locales with newly added resources
        if len(new_locales):
            pretranslate(db_project, locales=new_locales)

        locales = db_project.locales.exclude(pk__in=new_locales).values_list(
            "pk", flat=True
        )

        # Pretranslate newly added entities for all locales
        if new_entities and locales:
            new_entities = list(set(new_entities))
            pretranslate(db_project, locales=locales, entities=new_entities)
Ejemplo n.º 10
0
def sync_project_repo(self,
                      project_pk,
                      repo_pk,
                      project_sync_log_pk,
                      now,
                      obsolete_vcs_entities=None,
                      obsolete_vcs_resources=None,
                      new_paths=None,
                      locale=None,
                      no_pull=False,
                      no_commit=False,
                      full_scan=False):
    db_project = get_or_fail(
        Project,
        pk=project_pk,
        message='Could not sync project with pk={0}, not found.'.format(
            project_pk))
    repo = get_or_fail(
        Repository,
        pk=repo_pk,
        message='Could not sync repo with pk={0}, not found.'.format(repo_pk))
    project_sync_log = get_or_fail(
        ProjectSyncLog,
        pk=project_sync_log_pk,
        message=(
            'Could not sync project {0}, log with pk={1} not found.'.format(
                db_project.slug, project_sync_log_pk)))
    log.info('Syncing repo: {}'.format(repo.url))

    repo_sync_log = RepositorySyncLog.objects.create(
        project_sync_log=project_sync_log,
        repository=repo,
        start_time=timezone.now())

    # Pull VCS changes in case we're on a different worker than the one
    # sync started on.
    if not no_pull:
        pull_changes(db_project)

    if locale:
        locales = [locale]
    else:
        locales = repo.locales

    # Cannot skip earlier - repo.locales is only available after pull_changes()
    if not locales:
        log.debug(
            'Skipping repo `{0}` for project {1}, no locales to sync found within.'
            .format(repo.url, db_project.slug))
        repo_sync_log.end()
        return

    vcs_project = VCSProject(
        db_project,
        locales=locales,
        obsolete_entities_paths=Resource.objects.obsolete_entities_paths(
            obsolete_vcs_entities),
        new_paths=new_paths,
        full_scan=full_scan)

    for locale in locales:
        try:
            with transaction.atomic():
                # Skip locales that have nothing to sync
                if vcs_project.synced_locales and locale not in vcs_project.synced_locales:
                    if obsolete_vcs_entities or obsolete_vcs_resources:
                        update_locale_project_locale_stats(locale, db_project)
                    continue

                changeset = ChangeSet(db_project, vcs_project, now,
                                      obsolete_vcs_entities,
                                      obsolete_vcs_resources)
                update_translations(db_project, vcs_project, locale, changeset)
                changeset.execute()

                update_translated_resources(db_project, vcs_project, changeset,
                                            locale)

                # Skip if none of the locales has anything to sync
                # VCSProject.synced_locales is set on a first call to
                # VCSProject.resources, which is set in
                # pontoon.sync.core.update_translated_resources()
                if len(vcs_project.synced_locales) == 0:
                    if obsolete_vcs_entities or obsolete_vcs_resources:
                        for l in locales:
                            update_locale_project_locale_stats(l, db_project)
                        db_project.aggregate_stats()

                    log.info(
                        'Skipping repo `{0}` for project {1}, none of the locales has anything to sync.'
                        .format(repo.url, db_project.slug))
                    repo_sync_log.end()
                    return

                update_locale_project_locale_stats(locale, db_project)

                # Clear out the "has_changed" markers now that we've finished
                # syncing.
                (ChangedEntityLocale.objects.filter(
                    entity__resource__project=db_project,
                    locale=locale,
                    when__lte=now).delete())

                # Clean up any duplicate approvals at the end of sync right
                # before we commit the transaction to avoid race conditions.
                with connection.cursor() as cursor:
                    cursor.execute(
                        """
                        UPDATE base_translation AS b
                        SET approved = FALSE, approved_date = NULL
                        WHERE
                          id IN
                            (SELECT trans.id FROM base_translation AS trans
                             LEFT JOIN base_entity AS ent ON ent.id = trans.entity_id
                             LEFT JOIN base_resource AS res ON res.id = ent.resource_id
                             WHERE locale_id = %(locale_id)s
                               AND res.project_id = %(project_id)s)
                          AND approved_date !=
                            (SELECT max(approved_date)
                             FROM base_translation
                             WHERE entity_id = b.entity_id
                               AND locale_id = b.locale_id
                               AND (plural_form = b.plural_form OR plural_form IS NULL));
                    """, {
                            'locale_id': locale.id,
                            'project_id': db_project.id
                        })

                # Perform the commit last so that, if it succeeds, there is
                # nothing after it to fail.
                if not no_commit and locale in changeset.locales_to_commit:
                    commit_changes(db_project, vcs_project, changeset, locale)

        except CommitToRepositoryException as err:
            # Transaction aborted, log and move on to the next locale.
            log.warning(
                'Failed to sync locale {locale} for project {project} due to '
                'commit error: {error}'.format(
                    locale=locale.code,
                    project=db_project.slug,
                    error=err,
                ))

    with transaction.atomic():
        db_project.aggregate_stats()

    log.info('Synced translations for project {0} in locales {1}.'.format(
        db_project.slug,
        ','.join(locale.code for locale in vcs_project.synced_locales)))
    repo_sync_log.end()
Ejemplo n.º 11
0
def sync_translations(self,
                      project_pk,
                      project_sync_log_pk,
                      now,
                      project_changes=None,
                      obsolete_vcs_resources=None,
                      new_paths=None,
                      locale=None,
                      no_pull=False,
                      no_commit=False,
                      full_scan=False):
    db_project = get_or_fail(
        Project,
        pk=project_pk,
        message='Could not sync project with pk={0}, not found.'.format(
            project_pk))

    repos = db_project.translation_repositories()
    repo_pk = repos[0].pk
    repo = get_or_fail(
        Repository,
        pk=repo_pk,
        message='Could not sync repo with pk={0}, not found.'.format(repo_pk))

    project_sync_log = get_or_fail(
        ProjectSyncLog,
        pk=project_sync_log_pk,
        message=(
            'Could not sync project {0}, log with pk={1} not found.'.format(
                db_project.slug, project_sync_log_pk)))

    log.info('Syncing translations for project: {}'.format(db_project.slug))

    repo_sync_log = RepositorySyncLog.objects.create(
        project_sync_log=project_sync_log,
        repository=repo,
        start_time=timezone.now())

    if locale:
        locales = db_project.locales.filter(pk=locale.pk)
    else:
        locales = db_project.locales.all()

    if not locales:
        log.info(
            'Skipping syncing translations for project {0}, no locales to sync found within.'
            .format(db_project.slug))
        repo_sync_log.end()
        return

    # If project repositories have API access, we can retrieve latest commit hashes and detect
    # changed locales before the expensive VCS pull/clone operations. When performing full scan,
    # we still need to sync all locales.
    if not full_scan:
        locales = get_changed_locales(db_project, locales, now)

    # Pull VCS changes in case we're on a different worker than the one
    # sync started on.
    if not no_pull:
        log.info('Pulling changes for project {0} started.'.format(
            db_project.slug))
        repos_changed, repo_locales = pull_changes(db_project, locales)
        repos = repos.filter(pk__in=repo_locales.keys())
        log.info('Pulling changes for project {0} complete.'.format(
            db_project.slug))

    changed_resources = []
    obsolete_vcs_entities = []

    if project_changes:
        updated_entity_pks = []
        for locale_code, db_entity, vcs_entity in project_changes['update_db']:
            updated_entity_pks.append(db_entity.pk)

        obsolete_entity_pks = project_changes['obsolete_db']
        changed_resources = db_project.resources.filter(
            Q(entities__date_created=now)
            | Q(entities__pk__in=updated_entity_pks +
                obsolete_entity_pks)).distinct()

        obsolete_vcs_entities = project_changes['obsolete_db']

    # If none of the repos has changed since the last sync and there are
    # no Pontoon-side changes for this project, quit early.
    if (not full_scan and not db_project.needs_sync and not repos_changed
            and not (changed_resources or obsolete_vcs_resources)):
        log.info('Skipping project {0}, no changes detected.'.format(
            db_project.slug))
        repo_sync_log.end()
        return

    obsolete_entities_paths = (
        Resource.objects.obsolete_entities_paths(obsolete_vcs_entities)
        if obsolete_vcs_entities else None)

    vcs_project = VCSProject(db_project,
                             now,
                             locales=locales,
                             repo_locales=repo_locales,
                             obsolete_entities_paths=obsolete_entities_paths,
                             new_paths=new_paths,
                             full_scan=full_scan)

    synced_locales = set()
    failed_locales = set()

    for locale in locales:
        try:
            with transaction.atomic():
                # Sets VCSProject.synced_locales, needed to skip early
                if not vcs_project.synced_locales:
                    vcs_project.resources

                # Skip all locales if none of the them has anything to sync
                if len(vcs_project.synced_locales) == 0:
                    break

                # Skip locales that have nothing to sync
                if vcs_project.synced_locales and locale not in vcs_project.synced_locales:
                    continue

                changeset = ChangeSet(db_project, vcs_project, now, locale)
                update_translations(db_project, vcs_project, locale, changeset)
                changeset.execute()
                update_translated_resources(db_project, vcs_project, locale)
                update_locale_project_locale_stats(locale, db_project)

                # Clear out the "has_changed" markers now that we've finished
                # syncing.
                (ChangedEntityLocale.objects.filter(
                    entity__resource__project=db_project,
                    locale=locale,
                    when__lte=now).delete())

                # Clean up any duplicate approvals at the end of sync right
                # before we commit the transaction to avoid race conditions.
                with connection.cursor() as cursor:
                    cursor.execute(
                        """
                        UPDATE base_translation AS b
                        SET approved = FALSE, approved_date = NULL
                        WHERE
                          id IN
                            (SELECT trans.id FROM base_translation AS trans
                             LEFT JOIN base_entity AS ent ON ent.id = trans.entity_id
                             LEFT JOIN base_resource AS res ON res.id = ent.resource_id
                             WHERE locale_id = %(locale_id)s
                               AND res.project_id = %(project_id)s)
                          AND approved_date !=
                            (SELECT max(approved_date)
                             FROM base_translation
                             WHERE entity_id = b.entity_id
                               AND locale_id = b.locale_id
                               AND (plural_form = b.plural_form OR plural_form IS NULL));
                    """, {
                            'locale_id': locale.id,
                            'project_id': db_project.id
                        })

                # Perform the commit last so that, if it succeeds, there is
                # nothing after it to fail.
                if not no_commit and locale in changeset.locales_to_commit:
                    commit_changes(db_project, vcs_project, changeset, locale)

                log.info(
                    'Synced locale {locale} for project {project}.'.format(
                        locale=locale.code,
                        project=db_project.slug,
                    ))

                synced_locales.add(locale.code)

        except CommitToRepositoryException as err:
            # Transaction aborted, log and move on to the next locale.
            log.warning(
                'Failed to sync locale {locale} for project {project} due to '
                'commit error: {error}'.format(
                    locale=locale.code,
                    project=db_project.slug,
                    error=err,
                ))

            failed_locales.add(locale.code)

    # If sources have changed, update stats for all locales.
    if changed_resources or obsolete_vcs_resources:
        for locale in db_project.locales.all():
            # Already synced.
            if locale.code in synced_locales:
                continue

            # We have files: update all translated resources.
            if locale in locales:
                update_translated_resources(db_project, vcs_project, locale)

            # We don't have files: we can still update asymmetric translated resources.
            else:
                update_translated_resources_no_files(db_project, locale,
                                                     changed_resources)

            update_locale_project_locale_stats(locale, db_project)
            synced_locales.add(locale.code)

            log.info(
                'Synced source changes for locale {locale} for project {project}.'
                .format(
                    locale=locale.code,
                    project=db_project.slug,
                ))

        db_project.aggregate_stats()

    if synced_locales:
        log.info('Synced translations for project {0} in locales {1}.'.format(
            db_project.slug, ','.join(synced_locales)))
    elif failed_locales:
        log.info(
            'Failed to sync translations for project {0} due to commit error.'.
            format(db_project.slug))
    else:
        log.info(
            'Skipping syncing translations for project {0}, none of the locales '
            'has anything to sync.'.format(db_project.slug))

    for r in repos:
        r.set_last_synced_revisions(locales=repo_locales[r.pk].exclude(
            code__in=failed_locales))
    repo_sync_log.end()
Ejemplo n.º 12
0
def handle_upload_content(slug, code, part, f, user):
    """
    Update translations in the database from uploaded file.

    :arg str slug: Project slug.
    :arg str code: Locale code.
    :arg str part: Resource path or Subpage name.
    :arg UploadedFile f: UploadedFile instance.
    :arg User user: User uploading the file.
    """
    # Avoid circular import; someday we should refactor to avoid.
    from pontoon.sync import formats
    from pontoon.sync.changeset import ChangeSet
    from pontoon.sync.vcs.models import VCSProject
    from pontoon.base.models import (
        ChangedEntityLocale,
        Entity,
        Locale,
        Project,
        Resource,
        TranslatedResource,
        Translation,
    )

    relative_path = _get_relative_path_from_part(slug, part)
    project = get_object_or_404(Project, slug=slug)
    locale = get_object_or_404(Locale, code=code)
    resource = get_object_or_404(Resource,
                                 project__slug=slug,
                                 path=relative_path)

    # Store uploaded file to a temporary file and parse it
    extension = os.path.splitext(f.name)[1]
    with tempfile.NamedTemporaryFile(
            prefix="strings" if extension == ".xml" else "",
            suffix=extension,
    ) as temp:
        for chunk in f.chunks():
            temp.write(chunk)
        temp.flush()
        resource_file = formats.parse(temp.name)

    # Update database objects from file
    changeset = ChangeSet(project, VCSProject(project, locales=[locale]),
                          timezone.now())
    entities_qs = (Entity.objects.filter(
        resource__project=project,
        resource__path=relative_path,
        obsolete=False).prefetch_related(
            Prefetch(
                "translation_set",
                queryset=Translation.objects.filter(locale=locale),
                to_attr="db_translations",
            )).prefetch_related(
                Prefetch(
                    "translation_set",
                    queryset=Translation.objects.filter(
                        locale=locale, approved_date__lte=timezone.now()),
                    to_attr="db_translations_approved_before_sync",
                )))
    entities_dict = {entity.key: entity for entity in entities_qs}

    for vcs_translation in resource_file.translations:
        key = vcs_translation.key
        if key in entities_dict:
            entity = entities_dict[key]
            changeset.update_entity_translations_from_vcs(
                entity,
                locale.code,
                vcs_translation,
                user,
                entity.db_translations,
                entity.db_translations_approved_before_sync,
            )

    changeset.bulk_create_translations()
    changeset.bulk_update_translations()
    changeset.bulk_log_actions()

    if changeset.changed_translations:
        # Update 'active' status of all changed translations and their siblings,
        # i.e. translations of the same entity to the same locale.
        changed_pks = {t.pk for t in changeset.changed_translations}
        (Entity.objects.filter(
            translation__pk__in=changed_pks).reset_active_translations(
                locale=locale))

        # Run checks and create TM entries for translations that pass them
        valid_translations = changeset.bulk_check_translations()
        changeset.bulk_create_translation_memory_entries(valid_translations)

        # Remove any TM entries of translations that got rejected
        changeset.bulk_remove_translation_memory_entries()

    TranslatedResource.objects.get(resource=resource,
                                   locale=locale).calculate_stats()

    # Mark translations as changed
    changed_entities = {}
    existing = ChangedEntityLocale.objects.values_list("entity",
                                                       "locale").distinct()
    for t in changeset.changed_translations:
        key = (t.entity.pk, t.locale.pk)
        # Remove duplicate changes to prevent unique constraint violation
        if key not in existing:
            changed_entities[key] = ChangedEntityLocale(entity=t.entity,
                                                        locale=t.locale)

    ChangedEntityLocale.objects.bulk_create(changed_entities.values())

    # Update latest translation
    if changeset.translations_to_create:
        changeset.translations_to_create[-1].update_latest_translation()
Ejemplo n.º 13
0
    def setUp(self):
        self.now = aware_datetime(1970, 1, 1)

        timezone_patch = patch("pontoon.sync.tasks.timezone")
        self.mock_timezone = timezone_patch.start()
        self.addCleanup(timezone_patch.stop)
        self.mock_timezone.now.return_value = self.now

        self.translated_locale = LocaleFactory.create(code="translated-locale")
        self.inactive_locale = LocaleFactory.create(code="inactive-locale")
        self.repository = RepositoryFactory()

        self.db_project = ProjectFactory.create(
            name="db-project",
            locales=[self.translated_locale],
            repositories=[self.repository],
        )
        self.main_db_resource = ResourceFactory.create(project=self.db_project,
                                                       path="main.lang",
                                                       format="lang")
        self.other_db_resource = ResourceFactory.create(
            project=self.db_project, path="other.lang", format="lang")
        self.missing_db_resource = ResourceFactory.create(
            project=self.db_project, path="missing.lang", format="lang")
        self.main_db_entity = EntityFactory.create(
            resource=self.main_db_resource,
            string="Source String",
            key="Source String",
            obsolete=False,
        )
        self.other_db_entity = EntityFactory.create(
            resource=self.other_db_resource,
            string="Other Source String",
            key="Other Source String",
            obsolete=False,
        )
        self.main_db_translation = TranslationFactory.create(
            entity=self.main_db_entity,
            plural_form=None,
            locale=self.translated_locale,
            string="Translated String",
            date=aware_datetime(1970, 1, 1),
            approved=True,
            extra={"tags": []},
        )

        # Load paths from the fake locale directory.
        checkout_path_patch = patch.object(
            Project,
            "checkout_path",
            new_callable=PropertyMock,
            return_value=FAKE_CHECKOUT_PATH,
        )
        checkout_path_patch.start()

        self.addCleanup(checkout_path_patch.stop)

        vcs_changed_files = {
            self.main_db_resource.path: [self.translated_locale],
            self.other_db_resource.path: [self.translated_locale],
            self.missing_db_resource.path: [self.translated_locale],
        }

        changed_files_patch = patch.object(
            VCSProject,
            "changed_files",
            new_callable=PropertyMock,
            return_value=vcs_changed_files,
        )
        changed_files_patch.start()
        self.addCleanup(changed_files_patch.stop)

        source_repository = patch.object(
            Project,
            "source_repository",
            new_callable=PropertyMock,
            return_value=self.db_project.repositories.all()[0],
        )
        source_repository.start()
        self.addCleanup(source_repository.stop)

        self.vcs_project = VCSProject(self.db_project)
        self.main_vcs_resource = self.vcs_project.resources[
            self.main_db_resource.path]
        self.other_vcs_resource = self.vcs_project.resources[
            self.other_db_resource.path]
        self.missing_vcs_resource = self.vcs_project.resources[
            self.missing_db_resource.path]
        self.main_vcs_entity = self.main_vcs_resource.entities["Source String"]
        self.main_vcs_translation = self.main_vcs_entity.translations[
            "translated-locale"]

        # Mock VCSResource.save() for each resource to avoid altering
        # the filesystem.
        resource_save_patch = patch.object(VCSResource, "save")
        resource_save_patch.start()
        self.addCleanup(resource_save_patch.stop)

        self.changeset = ChangeSet(
            self.db_project,
            self.vcs_project,
            aware_datetime(1970, 1, 1),
            self.translated_locale,
        )
Ejemplo n.º 14
0
def sync_project_repo(self, project_pk, repo_pk, project_sync_log_pk, now,
                      no_pull=False, no_commit=False):
    db_project = get_or_fail(Project, pk=project_pk,
        message='Could not sync project with pk={0}, not found.'.format(project_pk))
    repo = get_or_fail(Repository, pk=repo_pk,
        message='Could not sync repo with pk={0}, not found.'.format(project_pk))
    project_sync_log = get_or_fail(ProjectSyncLog, pk=project_sync_log_pk,
        message=('Could not sync project {0}, log with pk={1} not found.'
                 .format(db_project.slug, project_sync_log_pk)))

    repo_sync_log = RepositorySyncLog.objects.create(
        project_sync_log=project_sync_log,
        repository=repo,
        start_time=timezone.now()
    )

    # Pull VCS changes in case we're on a different worker than the one
    # sync started on.
    if not no_pull:
        pull_changes(db_project)

    if len(repo.locales) < 1:
        log.warning('Could not sync repo `{0}`, no locales found within.'
                    .format(repo.url))
        repo_sync_log.end_time = timezone.now()
        repo_sync_log.save(update_fields=['end_time'])
        return

    vcs_project = VCSProject(db_project, locales=repo.locales)
    for locale in repo.locales:
        try:
            with transaction.atomic():
                changeset = ChangeSet(db_project, vcs_project, now)
                update_translations(db_project, vcs_project, locale, changeset)
                changeset.execute()

                update_project_stats(db_project, vcs_project, changeset, locale)

                # Clear out the "has_changed" markers now that we've finished
                # syncing.
                (ChangedEntityLocale.objects
                    .filter(entity__resource__project=db_project,
                            locale=locale,
                            when__lte=now)
                    .delete())
                db_project.has_changed = False
                db_project.save(update_fields=['has_changed'])

                # Clean up any duplicate approvals at the end of sync right
                # before we commit the transaction to avoid race conditions.
                with connection.cursor() as cursor:
                    cursor.execute("""
                        UPDATE base_translation AS b
                        SET approved = FALSE, approved_date = NULL
                        WHERE
                          id IN
                            (SELECT trans.id FROM base_translation AS trans
                             LEFT JOIN base_entity AS ent ON ent.id = trans.entity_id
                             LEFT JOIN base_resource AS res ON res.id = ent.resource_id
                             WHERE locale_id = %(locale_id)s
                               AND res.project_id = %(project_id)s)
                          AND approved_date !=
                            (SELECT max(approved_date)
                             FROM base_translation
                             WHERE entity_id = b.entity_id
                               AND locale_id = b.locale_id
                               AND (plural_form = b.plural_form OR plural_form IS NULL));
                    """, {
                        'locale_id': locale.id,
                        'project_id': db_project.id
                    })

                # Perform the commit last so that, if it succeeds, there is
                # nothing after it to fail.
                if not no_commit and locale in changeset.locales_to_commit:
                    commit_changes(db_project, vcs_project, changeset, locale)
        except CommitToRepositoryException as err:
            # Transaction aborted, log and move on to the next locale.
            log.warning(
                'Failed to sync locale {locale} for project {project} due to '
                'commit error: {error}'.format(
                    locale=locale.code,
                    project=db_project.slug,
                    error=err,
                )
            )

    repo_sync_log.end_time = timezone.now()
    repo_sync_log.save()
    log.info('Synced translations for project {0} in locales {1}.'.format(
        db_project.slug, ','.join(locale.code for locale in repo.locales)
    ))
Ejemplo n.º 15
0
def sync_translations(self, project_pk, repo_pk, project_sync_log_pk, now, project_changes=None,
                      obsolete_vcs_resources=None, new_paths=None, locale=None, no_pull=False, no_commit=False,
                      full_scan=False):
    db_project = get_or_fail(Project, pk=project_pk,
        message='Could not sync project with pk={0}, not found.'.format(project_pk))
    repo = get_or_fail(Repository, pk=repo_pk,
        message='Could not sync repo with pk={0}, not found.'.format(repo_pk))
    project_sync_log = get_or_fail(ProjectSyncLog, pk=project_sync_log_pk,
        message=('Could not sync project {0}, log with pk={1} not found.'
                 .format(db_project.slug, project_sync_log_pk)))

    log.info('Syncing translations for repo: {}'.format(repo.url))

    repo_sync_log = RepositorySyncLog.objects.create(
        project_sync_log=project_sync_log,
        repository=repo,
        start_time=timezone.now()
    )

    if locale:
        locales = [locale]
    else:
        locales = repo.locales

    if not locales:
        log.info('Skipping repo `{0}` for project {1}, no locales to sync found within.'
                  .format(repo.url, db_project.slug))
        repo_sync_log.end()
        return

    # Pull VCS changes in case we're on a different worker than the one
    # sync started on.
    if not no_pull:
        repos_changed = pull_changes(db_project)

    resources_changed = []
    obsolete_vcs_entities = []
    if project_changes:
        resources_changed = (
            project_changes['update_db'] +
            project_changes['obsolete_db'] +
            project_changes['create_db']
        )
        obsolete_vcs_entities = project_changes['obsolete_db']

    # If none of the repos has changed since the last sync and there are
    # no Pontoon-side changes for this project, quit early.
    if not full_scan and not db_project.needs_sync and not repos_changed and not (resources_changed or obsolete_vcs_resources):
        log.info('Skipping project {0}, no changes detected.'.format(db_project.slug))
        repo_sync_log.end()
        return

    obsolete_entities_paths = Resource.objects.obsolete_entities_paths(obsolete_vcs_entities) if obsolete_vcs_entities else None

    vcs_project = VCSProject(
        db_project,
        locales=locales,
        obsolete_entities_paths=obsolete_entities_paths,
        new_paths=new_paths,
        full_scan=full_scan
    )

    failed_locales = set()

    for locale in locales:
        try:
            with transaction.atomic():
                # Sets VCSProject.synced_locales, needed to skip early
                if not vcs_project.synced_locales:
                    vcs_project.resources

                if not obsolete_vcs_resources:
                    # Skip all locales if none of the them has anything to sync
                    if len(vcs_project.synced_locales) == 0:
                        if resources_changed:
                            for l in locales:
                                update_translated_resources(db_project, vcs_project, l)
                                update_locale_project_locale_stats(l, db_project)
                            db_project.aggregate_stats()

                        log.info('Skipping repo `{0}` for project {1}, none of the locales has anything to sync.'
                                 .format(repo.url, db_project.slug))
                        repo.set_last_synced_revisions()
                        repo_sync_log.end()
                        return

                    # Skip locales that have nothing to sync
                    if vcs_project.synced_locales and locale not in vcs_project.synced_locales:
                        if resources_changed:
                            update_translated_resources(db_project, vcs_project, locale)
                            update_locale_project_locale_stats(locale, db_project)
                            log.debug('Skipping locale `{0}` for project {1}, no changes detected.'
                                      .format(locale.code, db_project.slug))
                        continue

                changeset = ChangeSet(db_project, vcs_project, now, obsolete_vcs_entities, obsolete_vcs_resources, locale)
                update_translations(db_project, vcs_project, locale, changeset)
                changeset.execute()
                update_translated_resources(db_project, vcs_project, locale)
                update_locale_project_locale_stats(locale, db_project)

                # Clear out the "has_changed" markers now that we've finished
                # syncing.
                (ChangedEntityLocale.objects
                    .filter(entity__resource__project=db_project,
                            locale=locale,
                            when__lte=now)
                    .delete())

                # Clean up any duplicate approvals at the end of sync right
                # before we commit the transaction to avoid race conditions.
                with connection.cursor() as cursor:
                    cursor.execute("""
                        UPDATE base_translation AS b
                        SET approved = FALSE, approved_date = NULL
                        WHERE
                          id IN
                            (SELECT trans.id FROM base_translation AS trans
                             LEFT JOIN base_entity AS ent ON ent.id = trans.entity_id
                             LEFT JOIN base_resource AS res ON res.id = ent.resource_id
                             WHERE locale_id = %(locale_id)s
                               AND res.project_id = %(project_id)s)
                          AND approved_date !=
                            (SELECT max(approved_date)
                             FROM base_translation
                             WHERE entity_id = b.entity_id
                               AND locale_id = b.locale_id
                               AND (plural_form = b.plural_form OR plural_form IS NULL));
                    """, {
                        'locale_id': locale.id,
                        'project_id': db_project.id
                    })

                # Perform the commit last so that, if it succeeds, there is
                # nothing after it to fail.
                if not no_commit and locale in changeset.locales_to_commit:
                    commit_changes(db_project, vcs_project, changeset, locale)

                log.info(
                    'Synced locale {locale} for project {project}.'.format(
                        locale=locale.code,
                        project=db_project.slug,
                    )
                )

        except CommitToRepositoryException as err:
            # Transaction aborted, log and move on to the next locale.
            log.warning(
                'Failed to sync locale {locale} for project {project} due to '
                'commit error: {error}'.format(
                    locale=locale.code,
                    project=db_project.slug,
                    error=err,
                )
            )

            failed_locales.add(locale)

    with transaction.atomic():
        db_project.aggregate_stats()

    synced_locales = [locale.code for locale in (vcs_project.synced_locales - failed_locales)]

    if synced_locales:
        log.info('Synced translations for project {0} in locales {1}.'.format(
            db_project.slug, ','.join(synced_locales)
        ))
    else:
        log.info('Failed to sync translations for project {0} due to commit error.'.format(
            db_project.slug
        ))

    repo.set_last_synced_revisions(exclude=failed_locales)
    repo_sync_log.end()
Ejemplo n.º 16
0
def handle_upload_content(slug, code, part, f, user):
    """
    Update translations in the database from uploaded file.

    :arg str slug: Project slug.
    :arg str code: Locale code.
    :arg str part: Resource path or Subpage name.
    :arg UploadedFile f: UploadedFile instance.
    :arg User user: User uploading the file.
    """
    # Avoid circular import; someday we should refactor to avoid.
    from pontoon.sync import formats
    from pontoon.sync.changeset import ChangeSet
    from pontoon.sync.vcs.models import VCSProject
    from pontoon.base.models import (
        ChangedEntityLocale,
        Entity,
        Locale,
        Project,
        Resource,
        TranslatedResource,
        Translation,
    )

    relative_path = _get_relative_path_from_part(slug, part)
    project = get_object_or_404(Project, slug=slug)
    locale = get_object_or_404(Locale, code=code)
    resource = get_object_or_404(Resource, project__slug=slug, path=relative_path)

    # Store uploaded file to a temporary file and parse it
    extension = os.path.splitext(f.name)[1]
    with tempfile.NamedTemporaryFile(
        prefix='strings' if extension == '.xml' else '',
        suffix=extension,
    ) as temp:
        for chunk in f.chunks():
            temp.write(chunk)
        temp.flush()
        resource_file = formats.parse(temp.name)

    # Update database objects from file
    changeset = ChangeSet(
        project,
        VCSProject(project, locales=[locale]),
        timezone.now()
    )
    entities_qs = Entity.objects.filter(
        resource__project=project,
        resource__path=relative_path,
        obsolete=False
    ).prefetch_related(
        Prefetch(
            'translation_set',
            queryset=Translation.objects.filter(locale=locale),
            to_attr='db_translations'
        )
    ).prefetch_related(
        Prefetch(
            'translation_set',
            queryset=Translation.objects.filter(locale=locale, approved_date__lte=timezone.now()),
            to_attr='db_translations_approved_before_sync'
        )
    )
    entities_dict = {entity.key: entity for entity in entities_qs}

    for vcs_translation in resource_file.translations:
        key = vcs_translation.key
        if key in entities_dict:
            entity = entities_dict[key]
            changeset.update_entity_translations_from_vcs(
                entity, locale.code, vcs_translation, user,
                entity.db_translations, entity.db_translations_approved_before_sync
            )

    changeset.bulk_create_translations()
    changeset.bulk_update_translations()

    if changeset.changed_translations:
        # Update 'active' status of all changed translations and their siblings,
        # i.e. translations of the same entity to the same locale.
        changed_pks = {t.pk for t in changeset.changed_translations}
        (
            Entity.objects
            .filter(translation__pk__in=changed_pks)
            .reset_active_translations(locale=locale)
        )

        # Run checks and create TM entries for translations that pass them
        valid_translations = changeset.bulk_check_translations()
        changeset.bulk_create_translation_memory_entries(valid_translations)

    TranslatedResource.objects.get(resource=resource, locale=locale).calculate_stats()

    # Mark translations as changed
    changed_entities = {}
    existing = ChangedEntityLocale.objects.values_list('entity', 'locale').distinct()
    for t in changeset.changed_translations:
        key = (t.entity.pk, t.locale.pk)
        # Remove duplicate changes to prevent unique constraint violation
        if key not in existing:
            changed_entities[key] = ChangedEntityLocale(entity=t.entity, locale=t.locale)

    ChangedEntityLocale.objects.bulk_create(changed_entities.values())

    # Update latest translation
    if changeset.translations_to_create:
        changeset.translations_to_create[-1].update_latest_translation()
Ejemplo n.º 17
0
def sync_translations(
    self, project_pk, project_sync_log_pk, now, added_paths=None, removed_paths=None,
    changed_paths=None, locale=None, no_pull=False, no_commit=False, full_scan=False,
):
    db_project = get_or_fail(
        Project,
        pk=project_pk,
        message='Could not sync project with pk={0}, not found.'.format(project_pk)
    )

    repos = db_project.translation_repositories()
    repo_pk = repos[0].pk
    repo = get_or_fail(
        Repository,
        pk=repo_pk,
        message='Could not sync repo with pk={0}, not found.'.format(repo_pk)
    )

    project_sync_log = get_or_fail(
        ProjectSyncLog,
        pk=project_sync_log_pk,
        message=(
            'Could not sync project {0}, log with pk={1} not found.'
            .format(db_project.slug, project_sync_log_pk)
        )
    )

    log.info('Syncing translations for project: {}'.format(db_project.slug))

    repo_sync_log = RepositorySyncLog.objects.create(
        project_sync_log=project_sync_log,
        repository=repo,
        start_time=timezone.now()
    )

    if locale:
        locales = db_project.locales.filter(pk=locale.pk)
    else:
        locales = db_project.locales.all()

    if not locales:
        log.info('Skipping syncing translations for project {0}, no locales to sync found within.'
                 .format(db_project.slug))
        repo_sync_log.end()
        return

    # If project repositories have API access, we can retrieve latest commit hashes and detect
    # changed locales before the expensive VCS pull/clone operations. When performing full scan,
    # we still need to sync all locales.
    if not full_scan:
        locales = get_changed_locales(db_project, locales, now)

    readonly_locales = db_project.locales.filter(project_locale__readonly=True)
    added_and_changed_resources = db_project.resources.filter(
        path__in=list(added_paths or []) + list(changed_paths or [])
    ).distinct()

    # We should also sync files for which source file change - but only for read-only locales.
    # See bug 1372151 for more details.
    if added_and_changed_resources:
        changed_locales_pks = [l.pk for l in locales]
        readonly_locales_pks = [l.pk for l in readonly_locales]
        locales = db_project.locales.filter(
            pk__in=changed_locales_pks + readonly_locales_pks
        )

    # Pull VCS changes in case we're on a different worker than the one
    # sync started on.
    if not no_pull:
        log.info('Pulling changes for project {0} started.'.format(db_project.slug))
        repos_changed, repo_locales = pull_changes(db_project, locales)
        repos = repos.filter(pk__in=repo_locales.keys())
        log.info('Pulling changes for project {0} complete.'.format(db_project.slug))

    # If none of the repos has changed since the last sync and there are
    # no Pontoon-side changes for this project, quit early.
    if (
        not full_scan and
        not db_project.needs_sync and
        not repos_changed and
        not (added_paths or removed_paths or changed_paths)
    ):
        log.info('Skipping project {0}, no changes detected.'.format(db_project.slug))
        repo_sync_log.end()
        return

    vcs_project = VCSProject(
        db_project,
        now,
        locales=locales,
        repo_locales=repo_locales,
        added_paths=added_paths,
        changed_paths=changed_paths,
        full_scan=full_scan
    )

    synced_locales = set()
    failed_locales = set()

    for locale in locales:
        try:
            with transaction.atomic():
                # Sets VCSProject.synced_locales, needed to skip early
                if not vcs_project.synced_locales:
                    vcs_project.resources

                # Skip all locales if none of the them has anything to sync
                if len(vcs_project.synced_locales) == 0:
                    break

                # Skip locales that have nothing to sync
                if vcs_project.synced_locales and locale not in vcs_project.synced_locales:
                    continue

                changeset = ChangeSet(db_project, vcs_project, now, locale)
                update_translations(db_project, vcs_project, locale, changeset)
                changeset.execute()
                update_translated_resources(db_project, vcs_project, locale)
                update_locale_project_locale_stats(locale, db_project)

                # Clear out the "has_changed" markers now that we've finished
                # syncing.
                (ChangedEntityLocale.objects
                    .filter(entity__resource__project=db_project,
                            locale=locale,
                            when__lte=now)
                    .delete())

                # Perform the commit last so that, if it succeeds, there is
                # nothing after it to fail.
                if (
                    not no_commit and
                    locale in changeset.locales_to_commit and
                    locale not in readonly_locales
                ):
                    commit_changes(db_project, vcs_project, changeset, locale)

                log.info(
                    'Synced locale {locale} for project {project}.'.format(
                        locale=locale.code,
                        project=db_project.slug,
                    )
                )

                synced_locales.add(locale.code)

        except CommitToRepositoryException as err:
            # Transaction aborted, log and move on to the next locale.
            log.warning(
                'Failed to sync locale {locale} for project {project} due to '
                'commit error: {error}'.format(
                    locale=locale.code,
                    project=db_project.slug,
                    error=err,
                )
            )

            failed_locales.add(locale.code)

    # If sources have changed, update stats for all locales.
    if added_paths or removed_paths or changed_paths:
        for locale in db_project.locales.all():
            # Already synced.
            if locale.code in synced_locales:
                continue

            # We have files: update all translated resources.
            if locale in locales:
                update_translated_resources(db_project, vcs_project, locale)

            # We don't have files: we can still update asymmetric translated resources.
            else:
                update_translated_resources_no_files(
                    db_project,
                    locale,
                    added_and_changed_resources,
                )

            update_locale_project_locale_stats(locale, db_project)
            synced_locales.add(locale.code)

            log.info(
                'Synced source changes for locale {locale} for project {project}.'.format(
                    locale=locale.code,
                    project=db_project.slug,
                )
            )

        db_project.aggregate_stats()

    if synced_locales:
        log.info('Synced translations for project {0} in locales {1}.'.format(
            db_project.slug, ','.join(synced_locales)
        ))
    elif failed_locales:
        log.info('Failed to sync translations for project {0} due to commit error.'.format(
            db_project.slug
        ))
    else:
        log.info(
            'Skipping syncing translations for project {0}, none of the locales '
            'has anything to sync.'.format(db_project.slug)
        )

    for r in repos:
        r.set_last_synced_revisions(
            locales=repo_locales[r.pk].exclude(code__in=failed_locales)
        )
    repo_sync_log.end()
Ejemplo n.º 18
0
def sync_project_repo(self, project_pk, repo_pk, project_sync_log_pk, now, obsolete_vcs_entities=None,
                      obsolete_vcs_resources=None, locale=None, no_pull=False, no_commit=False,
                      full_scan=False):
    db_project = get_or_fail(Project, pk=project_pk,
        message='Could not sync project with pk={0}, not found.'.format(project_pk))
    repo = get_or_fail(Repository, pk=repo_pk,
        message='Could not sync repo with pk={0}, not found.'.format(repo_pk))
    project_sync_log = get_or_fail(ProjectSyncLog, pk=project_sync_log_pk,
        message=('Could not sync project {0}, log with pk={1} not found.'
                 .format(db_project.slug, project_sync_log_pk)))
    log.info('Syncing repo: {}'.format(repo.url))

    repo_sync_log = RepositorySyncLog.objects.create(
        project_sync_log=project_sync_log,
        repository=repo,
        start_time=timezone.now()
    )

    # Pull VCS changes in case we're on a different worker than the one
    # sync started on.
    if not no_pull:
        pull_changes(db_project)

    if locale:
        locales = [locale]
    else:
        locales = repo.locales

    vcs_project = VCSProject(
        db_project,
        locales=locales,
        obsolete_entities_paths=Resource.objects.obsolete_entities_paths(obsolete_vcs_entities),
        full_scan=full_scan
    )

    for locale in locales:
        try:
            with transaction.atomic():
                # Skip locales that have nothing to sync
                if vcs_project.synced_locales and locale not in vcs_project.synced_locales:
                    continue

                changeset = ChangeSet(db_project, vcs_project, now, obsolete_vcs_entities, obsolete_vcs_resources)
                update_translations(db_project, vcs_project, locale, changeset)
                changeset.execute()

                update_translated_resources(db_project, vcs_project, changeset, locale)

                # Skip if none of the locales has anything to sync
                # VCSProject.synced_locales is set on a first call to
                # VCSProject.resources, which is set in
                # pontoon.sync.core.update_translated_resources()
                if len(vcs_project.synced_locales) == 0:
                    log.info('Skipping repo `{0}` for project {1}, none of the locales has anything to sync.'
                             .format(repo.url, db_project.slug))
                    end_repo_sync(repo, repo_sync_log)
                    return

                locale.aggregate_stats()
                locale.project_locale.get(project=db_project).aggregate_stats()

                # Clear out the "has_changed" markers now that we've finished
                # syncing.
                (ChangedEntityLocale.objects
                    .filter(entity__resource__project=db_project,
                            locale=locale,
                            when__lte=now)
                    .delete())
                db_project.has_changed = False
                db_project.save(update_fields=['has_changed'])

                # Clean up any duplicate approvals at the end of sync right
                # before we commit the transaction to avoid race conditions.
                with connection.cursor() as cursor:
                    cursor.execute("""
                        UPDATE base_translation AS b
                        SET approved = FALSE, approved_date = NULL
                        WHERE
                          id IN
                            (SELECT trans.id FROM base_translation AS trans
                             LEFT JOIN base_entity AS ent ON ent.id = trans.entity_id
                             LEFT JOIN base_resource AS res ON res.id = ent.resource_id
                             WHERE locale_id = %(locale_id)s
                               AND res.project_id = %(project_id)s)
                          AND approved_date !=
                            (SELECT max(approved_date)
                             FROM base_translation
                             WHERE entity_id = b.entity_id
                               AND locale_id = b.locale_id
                               AND (plural_form = b.plural_form OR plural_form IS NULL));
                    """, {
                        'locale_id': locale.id,
                        'project_id': db_project.id
                    })

                # Perform the commit last so that, if it succeeds, there is
                # nothing after it to fail.
                if not no_commit and locale in changeset.locales_to_commit:
                    commit_changes(db_project, vcs_project, changeset, locale)

        except CommitToRepositoryException as err:
            # Transaction aborted, log and move on to the next locale.
            log.warning(
                'Failed to sync locale {locale} for project {project} due to '
                'commit error: {error}'.format(
                    locale=locale.code,
                    project=db_project.slug,
                    error=err,
                )
            )

    with transaction.atomic():
        db_project.aggregate_stats()

    log.info('Synced translations for project {0} in locales {1}.'.format(
        db_project.slug, ','.join(locale.code for locale in vcs_project.synced_locales)
    ))
    end_repo_sync(repo, repo_sync_log)
Ejemplo n.º 19
0
def sync_project(db_project, no_pull=False, no_commit=False):
    """
    Update the database with the current state of resources in version
    control and write any submitted translations from the database back
    to version control.
    """
    # Mark "now" at the start of sync to avoid messing with
    # translations submitted during sync.
    now = timezone.now()

    # Pull changes from VCS and update what we know about the files.
    if not no_pull:
        repos_changed = pull_changes(db_project)
    else:
        repos_changed = True  # Assume changed.

    # If the repos haven't changed since the last sync and there are
    # no Pontoon-side changes for this project, quit early.
    if not repos_changed and not db_project.needs_sync:
        log.info('Skipping project {0}, no changes detected.'.format(db_project.slug))
        return

    vcs_project = VCSProject(db_project)
    update_resources(db_project, vcs_project)

    # Collect all entities across VCS and the database and get their
    # keys so we can match up matching entities.
    vcs_entities = get_vcs_entities(vcs_project)
    db_entities = get_db_entities(db_project)
    entity_keys = set().union(db_entities.keys(), vcs_entities.keys())

    changeset = ChangeSet(db_project, vcs_project, now)
    for key in entity_keys:
        db_entity = db_entities.get(key, None)
        vcs_entity = vcs_entities.get(key, None)
        handle_entity(changeset, db_project, key, db_entity, vcs_entity)

    # Apply the changeset to the files, commit them, and update stats
    # entries in the DB.
    changeset.execute()
    if not no_commit:
        commit_changes(db_project, vcs_project, changeset)
    update_project_stats(db_project, vcs_project, changeset)

    # Clear out the "has_changed" markers now that we've finished
    # syncing.
    (ChangedEntityLocale.objects
        .filter(entity__resource__project=db_project, when__lte=now)
        .delete())
    db_project.has_changed = False
    db_project.save()

    # Clean up any duplicate approvals at the end of sync right
    # before we commit the transaction to avoid race conditions.
    with connection.cursor() as cursor:
        cursor.execute("""
            UPDATE base_translation AS b
            SET approved = FALSE, approved_date = NULL
            WHERE approved_date !=
                (SELECT max(approved_date)
                 FROM base_translation
                 WHERE entity_id = b.entity_id
                   AND locale_id = b.locale_id
                   AND (plural_form = b.plural_form OR plural_form IS NULL));
        """)

    log.info(u'Synced project {0}'.format(db_project.slug))
Ejemplo n.º 20
0
def handle_upload_content(slug, code, part, f, user):
    """
    Update translations in the database from uploaded file.

    :arg str slug: Project slug.
    :arg str code: Locale code.
    :arg str part: Resource path or Subpage name.
    :arg UploadedFile f: UploadedFile instance.
    :arg User user: User uploading the file.
    """
    # Avoid circular import; someday we should refactor to avoid.
    from pontoon.sync import formats
    from pontoon.sync.changeset import ChangeSet
    from pontoon.sync.vcs.models import VCSProject
    from pontoon.base.models import (
        ChangedEntityLocale,
        Entity,
        Locale,
        Project,
        Resource,
        TranslatedResource,
        Translation,
    )

    relative_path = _get_relative_path_from_part(slug, part)
    project = get_object_or_404(Project, slug=slug)
    locale = get_object_or_404(Locale, code=code)
    resource = get_object_or_404(Resource,
                                 project__slug=slug,
                                 path=relative_path)

    # Store uploaded file to a temporary file and parse it
    extension = os.path.splitext(f.name)[1]
    with tempfile.NamedTemporaryFile(suffix=extension) as temp:
        for chunk in f.chunks():
            temp.write(chunk)
        temp.flush()
        resource_file = formats.parse(temp.name)

    # Update database objects from file
    changeset = ChangeSet(project, VCSProject(project, locales=[locale]),
                          timezone.now())
    entities_qs = Entity.objects.filter(
        resource__project=project,
        resource__path=relative_path,
        obsolete=False).prefetch_related(
            Prefetch('translation_set',
                     queryset=Translation.objects.filter(locale=locale),
                     to_attr='db_translations')).prefetch_related(
                         Prefetch(
                             'translation_set',
                             queryset=Translation.objects.filter(
                                 locale=locale,
                                 approved_date__lte=timezone.now()),
                             to_attr='db_translations_approved_before_sync'))
    entities_dict = {entity.key: entity for entity in entities_qs}

    for vcs_translation in resource_file.translations:
        key = vcs_translation.key
        if key in entities_dict:
            entity = entities_dict[key]
            changeset.update_entity_translations_from_vcs(
                entity, locale.code, vcs_translation, user,
                entity.db_translations,
                entity.db_translations_approved_before_sync)

    changeset.bulk_create_translations()
    changeset.bulk_update_translations()
    changeset.bulk_create_translaton_memory_entries()
    TranslatedResource.objects.get(resource=resource,
                                   locale=locale).calculate_stats()

    # Mark translations as changed
    changed_entities = {}
    existing = ChangedEntityLocale.objects.values_list('entity',
                                                       'locale').distinct()
    for t in changeset.changed_translations:
        key = (t.entity.pk, t.locale.pk)
        # Remove duplicate changes to prevent unique constraint violation
        if key not in existing:
            changed_entities[key] = ChangedEntityLocale(entity=t.entity,
                                                        locale=t.locale)

    ChangedEntityLocale.objects.bulk_create(changed_entities.values())

    # Update latest translation
    if changeset.translations_to_create:
        changeset.translations_to_create[-1].update_latest_translation()
Ejemplo n.º 21
0
def sync_translations(
    self, project_pk, project_sync_log_pk, now, project_changes=None, obsolete_vcs_resources=None,
    new_paths=None, locale=None, no_pull=False, no_commit=False, full_scan=False
):
    db_project = get_or_fail(
        Project,
        pk=project_pk,
        message='Could not sync project with pk={0}, not found.'.format(project_pk)
    )

    repos = db_project.translation_repositories()
    repo_pk = repos[0].pk
    repo = get_or_fail(
        Repository,
        pk=repo_pk,
        message='Could not sync repo with pk={0}, not found.'.format(repo_pk)
    )

    project_sync_log = get_or_fail(
        ProjectSyncLog,
        pk=project_sync_log_pk,
        message=(
            'Could not sync project {0}, log with pk={1} not found.'
            .format(db_project.slug, project_sync_log_pk)
        )
    )

    log.info('Syncing translations for project: {}'.format(db_project.slug))

    repo_sync_log = RepositorySyncLog.objects.create(
        project_sync_log=project_sync_log,
        repository=repo,
        start_time=timezone.now()
    )

    if locale:
        locales = db_project.locales.filter(pk=locale.pk)
    else:
        locales = db_project.locales.all()

    if not locales:
        log.info('Skipping syncing translations for project {0}, no locales to sync found within.'
                 .format(db_project.slug))
        repo_sync_log.end()
        return

    # If project repositories have API access, we can retrieve latest commit hashes and detect
    # changed locales before the expensive VCS pull/clone operations. When performing full scan,
    # we still need to sync all locales.
    if not full_scan:
        locales = get_changed_locales(db_project, locales, now)

    # Pull VCS changes in case we're on a different worker than the one
    # sync started on.
    if not no_pull:
        log.info('Pulling changes for project {0} started.'.format(db_project.slug))
        repos_changed, repo_locales = pull_changes(db_project, locales)
        repos = repos.filter(pk__in=repo_locales.keys())
        log.info('Pulling changes for project {0} complete.'.format(db_project.slug))

    changed_resources = []
    obsolete_vcs_entities = []

    if project_changes:
        updated_entity_pks = []
        for locale_code, db_entity, vcs_entity in project_changes['update_db']:
            updated_entity_pks.append(db_entity.pk)

        obsolete_entity_pks = project_changes['obsolete_db']
        changed_resources = db_project.resources.filter(
            Q(entities__date_created=now) |
            Q(entities__pk__in=updated_entity_pks + obsolete_entity_pks)
        ).distinct()

        obsolete_vcs_entities = project_changes['obsolete_db']

    # If none of the repos has changed since the last sync and there are
    # no Pontoon-side changes for this project, quit early.
    if (
        not full_scan and
        not db_project.needs_sync and
        not repos_changed and
        not (changed_resources or obsolete_vcs_resources)
    ):
        log.info('Skipping project {0}, no changes detected.'.format(db_project.slug))
        repo_sync_log.end()
        return

    obsolete_entities_paths = (
        Resource.objects.obsolete_entities_paths(obsolete_vcs_entities) if obsolete_vcs_entities
        else None
    )

    vcs_project = VCSProject(
        db_project,
        now,
        locales=locales,
        repo_locales=repo_locales,
        obsolete_entities_paths=obsolete_entities_paths,
        new_paths=new_paths,
        full_scan=full_scan
    )

    synced_locales = set()
    failed_locales = set()

    for locale in locales:
        try:
            with transaction.atomic():
                # Sets VCSProject.synced_locales, needed to skip early
                if not vcs_project.synced_locales:
                    vcs_project.resources

                # Skip all locales if none of the them has anything to sync
                if len(vcs_project.synced_locales) == 0:
                    break

                # Skip locales that have nothing to sync
                if vcs_project.synced_locales and locale not in vcs_project.synced_locales:
                    continue

                changeset = ChangeSet(db_project, vcs_project, now, locale)
                update_translations(db_project, vcs_project, locale, changeset)
                changeset.execute()
                update_translated_resources(db_project, vcs_project, locale)
                update_locale_project_locale_stats(locale, db_project)

                # Clear out the "has_changed" markers now that we've finished
                # syncing.
                (ChangedEntityLocale.objects
                    .filter(entity__resource__project=db_project,
                            locale=locale,
                            when__lte=now)
                    .delete())

                # Clean up any duplicate approvals at the end of sync right
                # before we commit the transaction to avoid race conditions.
                with connection.cursor() as cursor:
                    cursor.execute("""
                        UPDATE base_translation AS b
                        SET approved = FALSE, approved_date = NULL
                        WHERE
                          id IN
                            (SELECT trans.id FROM base_translation AS trans
                             LEFT JOIN base_entity AS ent ON ent.id = trans.entity_id
                             LEFT JOIN base_resource AS res ON res.id = ent.resource_id
                             WHERE locale_id = %(locale_id)s
                               AND res.project_id = %(project_id)s)
                          AND approved_date !=
                            (SELECT max(approved_date)
                             FROM base_translation
                             WHERE entity_id = b.entity_id
                               AND locale_id = b.locale_id
                               AND (plural_form = b.plural_form OR plural_form IS NULL));
                    """, {
                        'locale_id': locale.id,
                        'project_id': db_project.id
                    })

                # Perform the commit last so that, if it succeeds, there is
                # nothing after it to fail.
                if not no_commit and locale in changeset.locales_to_commit:
                    commit_changes(db_project, vcs_project, changeset, locale)

                log.info(
                    'Synced locale {locale} for project {project}.'.format(
                        locale=locale.code,
                        project=db_project.slug,
                    )
                )

                synced_locales.add(locale.code)

        except CommitToRepositoryException as err:
            # Transaction aborted, log and move on to the next locale.
            log.warning(
                'Failed to sync locale {locale} for project {project} due to '
                'commit error: {error}'.format(
                    locale=locale.code,
                    project=db_project.slug,
                    error=err,
                )
            )

            failed_locales.add(locale.code)

    # If sources have changed, update stats for all locales.
    if changed_resources or obsolete_vcs_resources:
        for locale in db_project.locales.all():
            # Already synced.
            if locale.code in synced_locales:
                continue

            # We have files: update all translated resources.
            if locale in locales:
                update_translated_resources(db_project, vcs_project, locale)

            # We don't have files: we can still update asymmetric translated resources.
            else:
                update_translated_resources_no_files(db_project, locale, changed_resources)

            update_locale_project_locale_stats(locale, db_project)
            synced_locales.add(locale.code)

            log.info(
                'Synced source changes for locale {locale} for project {project}.'.format(
                    locale=locale.code,
                    project=db_project.slug,
                )
            )

        db_project.aggregate_stats()

    if synced_locales:
        log.info('Synced translations for project {0} in locales {1}.'.format(
            db_project.slug, ','.join(synced_locales)
        ))
    elif failed_locales:
        log.info('Failed to sync translations for project {0} due to commit error.'.format(
            db_project.slug
        ))
    else:
        log.info(
            'Skipping syncing translations for project {0}, none of the locales '
            'has anything to sync.'.format(db_project.slug)
        )

    for r in repos:
        r.set_last_synced_revisions(
            locales=repo_locales[r.pk].exclude(code__in=failed_locales)
        )
    repo_sync_log.end()