Esempio n. 1
0
def create_content(artifact_sha256, filename):
    """
    Creates PythonPackageContent from artifact.

    Args:
        artifact_sha256: validated artifact
        filename: file name
    Returns:
        queryset of the new created content
    """
    artifact = Artifact.objects.get(sha256=artifact_sha256)
    metadata = get_project_metadata_from_artifact(filename, artifact)

    data = parse_project_metadata(vars(metadata))
    data['packagetype'] = metadata.packagetype
    data['version'] = metadata.version
    data['filename'] = filename
    data['sha256'] = artifact.sha256

    @transaction.atomic()
    def create():
        content = PythonPackageContent.objects.create(**data)
        ContentArtifact.objects.create(artifact=artifact,
                                       content=content,
                                       relative_path=filename)
        return content

    new_content = create()
    resource = CreatedResource(content_object=new_content)
    resource.save()

    return PythonPackageContent.objects.filter(pk=new_content.pk)
Esempio n. 2
0
    def new_version(self, base_version=None):
        """
        Create a new RepositoryVersion for this Repository.

        Creation of a RepositoryVersion should be done in a RQ Job.

        Args:
            repository (pulpcore.app.models.Repository): to create a new version of
            base_version (pulpcore.app.models.RepositoryVersion): an optional repository version
                whose content will be used as the set of content for the new version

        Returns:
            pulpcore.app.models.RepositoryVersion: The Created RepositoryVersion

        """
        with transaction.atomic():
            version = RepositoryVersion(
                repository=self,
                number=int(self.next_version),
                base_version=base_version)
            version.save()

            if base_version:
                # first remove the content that isn't in the base version
                version.remove_content(version.content.exclude(pk__in=base_version.content))
                # now add any content that's in the base_version but not in version
                version.add_content(base_version.content.exclude(pk__in=version.content))

            if Task.current() and not self.sub_repo:
                resource = CreatedResource(content_object=version)
                resource.save()
            return version
Esempio n. 3
0
def migrate_repo_distributor(dist_migrator,
                             progress_dist,
                             pulp2dist,
                             repo_version=None):
    """
    Migrate repo distributor.

    Args:
        dist_migrator(Pulp2to3Distributor): distributor migrator class
        progress_dist: GroupProgressReport queryset for distribution creation
        pulp2dist(Pulp2Distributor): a pre-migrated distributor to migrate
        repo_version(RepositoryVersion): a pulp3 repo version
    """

    publication, distribution, created = dist_migrator.migrate_to_pulp3(
        pulp2dist, repo_version)
    if publication:
        pulp2dist.pulp3_publication = publication
    pulp2dist.pulp3_distribution = distribution
    pulp2dist.is_migrated = True
    pulp2dist.save()
    progress_dist.update(done=F('done') + 1)
    # CreatedResource were added  here because publications and repo versions
    # were listed among created resources and distributions were not. it could
    # create some confusion remotes are not listed still
    # TODO figure out what to do to make the output consistent
    if created:
        resource = CreatedResource(content_object=distribution)
        resource.save()
Esempio n. 4
0
    async def migrate_repo_distributor(pb,
                                       dist_migrator,
                                       pulp2dist,
                                       repo_version=None):
        """
        Migrate repo distributor.

        Args:
            dist_migrator(Pulp2to3Distributor): distributor migrator class
            pulp2dist(Pulp2Distributor): a pre-migrated distributor to migrate
            repo_version(RepositoryVersion): a pulp3 repo version
        """

        publication, distribution, created = await dist_migrator.migrate_to_pulp3(
            pulp2dist, repo_version)
        if publication:
            pulp2dist.pulp3_publication = publication
        if distribution:
            pulp2dist.pulp3_distribution = distribution
        pulp2dist.is_migrated = True
        pulp2dist.save()
        # CreatedResource were added  here because publications and repo versions
        # were listed among created resources and distributions were not. it could
        # create some confusion remotes are not listed still
        # TODO figure out what to do to make the output consistent
        if created:
            resource = CreatedResource(content_object=distribution)
            resource.save()
            pb.increment()
        else:
            pb.total -= 1
            pb.save()
Esempio n. 5
0
def tag_image(manifest_pk, tag, repository_pk):
    """
    Create a new repository version out of the passed tag name and the manifest.

    If the tag name is already associated with an existing manifest with the same digest,
    no new content is created. Note that a same tag name cannot be used for two different
    manifests. Due to this fact, an old Tag object is going to be removed from
    a new repository version when a manifest contains a digest which is not equal to the
    digest passed with POST request.
    """
    manifest = Manifest.objects.get(pk=manifest_pk)

    repository = Repository.objects.get(pk=repository_pk).cast()
    latest_version = repository.latest_version()

    tags_to_remove = Tag.objects.filter(pk__in=latest_version.content.all(), name=tag).exclude(
        tagged_manifest=manifest
    )

    manifest_tag, created = Tag.objects.get_or_create(name=tag, tagged_manifest=manifest)

    if created:
        resource = CreatedResource(content_object=manifest_tag)
        resource.save()
    else:
        manifest_tag.touch()

    tags_to_add = Tag.objects.filter(pk=manifest_tag.pk).exclude(
        pk__in=latest_version.content.all()
    )

    with repository.new_version() as repository_version:
        repository_version.remove_content(tags_to_remove)
        repository_version.add_content(tags_to_add)
Esempio n. 6
0
def upload_comps(tmp_file_id, repo_id=None, replace=False):
    """
    Upload comps.xml file.

    Args:
        tmp_file_id: uploaded comps.xml file.
        repo_id: repository primary key to associate incoming comps-content to.
        replace: if true, replace existing comps-related Content in the specified
            repository with those in the incoming comps.xml file.
    """
    temp_file = PulpTemporaryFile.objects.get(pk=tmp_file_id)
    created, all_objs = parse_comps_components(temp_file)

    for content in all_objs:
        crsrc = CreatedResource(content_object=content)
        crsrc.save()

    if repo_id:
        repository = RpmRepository.objects.get(pk=repo_id)
        if repository:
            all_ids = [obj.content_ptr_id for obj in all_objs]
            with repository.new_version() as new_version:
                if replace:
                    latest = repository.latest_version()
                    rmv_ids = latest.content.filter(pulp_type__in=(
                        PackageCategory.get_pulp_type(),
                        PackageEnvironment.get_pulp_type(),
                        PackageGroup.get_pulp_type(),
                        PackageLangpacks.get_pulp_type(),
                    ))
                    new_version.remove_content(
                        Content.objects.filter(pk__in=rmv_ids))
                new_version.add_content(Content.objects.filter(pk__in=all_ids))
Esempio n. 7
0
def migrate_from_pulp2(
    migration_plan_pk, validate=False, dry_run=False, skip_corrupted=False
):
    """
    Main task to migrate from Pulp 2 to Pulp 3.

    Schedule other tasks based on the specified Migration Plan.

    Args:
        migration_plan_pk (str): The migration plan PK.
        validate (bool): If True, don't migrate unless validation is successful.
        dry_run (bool): If True, nothing is migrated, only validation happens.
        skip_corrupted (bool): If True, corrupted content is skipped during migration,
                               no task failure.
    """

    # MongoDB connection initialization
    connection.initialize()

    plan = MigrationPlan.objects.get(pk=migration_plan_pk)
    missing_resources = plan.get_missing_resources()

    if (validate or dry_run) and missing_resources:
        raise PlanValidationError(
            "Validation failed: resources missing {}".format(missing_resources)
        )

    if dry_run:
        return

    task_group = TaskGroup(description="Migration Sub-tasks")
    task_group.save()
    GroupProgressReport(
        message="Repo version creation",
        code="create.repo_version",
        task_group=task_group,
    ).save()
    GroupProgressReport(
        message="Distribution creation",
        code="create.distribution",
        task_group=task_group,
    ).save()
    current_task = Task.current()
    current_task.task_group = task_group
    current_task.save()
    resource = CreatedResource(content_object=task_group)
    resource.save()

    # TODO: if plan is empty for a plugin, only migrate downloaded content

    pre_migrate_all_without_content(plan)
    pre_migrate_all_content(plan)
    handle_outdated_resources(plan)
    migrate_repositories(plan)
    migrate_importers(plan)
    migrate_content(plan, skip_corrupted=skip_corrupted)
    create_repoversions_publications_distributions(plan)

    task_group.finish()
Esempio n. 8
0
def create(*args, **kwargs):
    """
    Creates a :class:`~pulp_docker.app.models.DockerDistribution`.

    Raises:
        ValidationError: If the DockerDistributionSerializer is not valid

    """
    data = kwargs.pop('data', None)
    serializer = DockerDistributionSerializer(data=data)
    serializer.is_valid(raise_exception=True)
    serializer.save()
    resource = CreatedResource(content_object=serializer.instance)
    resource.save()
Esempio n. 9
0
def one_shot_upload(artifact_pk, filename, repository_pk=None):
    """
    One shot upload for pulp_python
    Args:
        artifact_pk: validated artifact
        filename: file name
        repository_pk: optional repository to add Content to
    """

    chart = {}
    with tempfile.TemporaryDirectory() as td:
        temp_path = os.path.join(td, filename)
        artifact = Artifact.objects.get(pk=artifact_pk)
        shutil.copy2(artifact.file.path, temp_path)

        with tarfile.open(temp_path) as tarball:
            chart_member = [
                m for m in tf.getmembers()
                if m.name.endswith('Chart.yaml') and m.name.count('/') == 1
            ]
            if len(chart_member) != 1:
                raise serializers.ValidationError('Unable to find Chart.yaml')

            chart_file = tarball.extractfile(chart_member)
            doc = yaml.load(chart_file, loader=yaml.SafeLoader)
            chart = {
                'name': doc['name'],
                'version': doc['version'],
                'digest': doc['digest'],

                # TODO: Handle multiple URLs better, maybe failover?
                'url': doc['urls'][0],
                'created': doc.get('created'),
                'app_version': doc.get('appVersion'),
                'description': doc.get('description'),
                'icon': doc.get('icon'),
                'keywords': doc.get('keywords', [])
            }

    new_content = ChartContent(**chart)
    new_content.save()

    if repository_pk:
        queryset = ChartContent.objects.filter(pk=new_content.pk)
        repository = ChartRepository.objects.get(pk=repository_pk)
        with repository.new_version() as new_version:
            new_version.add_content(queryset)

    resource = CreatedResource(content_object=new_content)
    resource.save()
Esempio n. 10
0
def create(*args, **kwargs):
    """
    Creates a :class:`~pulp_docker.app.models.DockerDistribution`.

    Raises:
        ValidationError: If the DockerDistributionSerializer is not valid

    """
    data = kwargs.pop('data', None)
    serializer = DockerDistributionSerializer(data=data)
    serializer.is_valid(raise_exception=True)
    serializer.save()
    resource = CreatedResource(content_object=serializer.instance)
    resource.save()
Esempio n. 11
0
def one_shot_upload(artifact_pk, filename, repository_pk=None):
    """
    One shot upload for pulp_python

    Args:
        artifact_pk: validated artifact
        filename: file name
        repository_pk: optional repository to add Content to
    """
    # iterate through extensions since splitext does not support things like .tar.gz
    for ext, packagetype in DIST_EXTENSIONS.items():
        if filename.endswith(ext):
            # Copy file to a temp directory under the user provided filename, we do this
            # because pkginfo validates that the filename has a valid extension before
            # reading it
            with tempfile.TemporaryDirectory() as td:
                temp_path = os.path.join(td, filename)
                artifact = Artifact.objects.get(pk=artifact_pk)
                shutil.copy2(artifact.file.path, temp_path)
                metadata = DIST_TYPES[packagetype](temp_path)
                metadata.packagetype = packagetype
                break
    else:
        raise serializers.ValidationError(
            _("Extension on {} is not a valid python extension "
              "(.whl, .exe, .egg, .tar.gz, .tar.bz2, .zip)").format(filename))
    data = parse_project_metadata(vars(metadata))
    data['classifiers'] = [{
        'name': classifier
    } for classifier in metadata.classifiers]
    data['packagetype'] = metadata.packagetype
    data['version'] = metadata.version
    data['filename'] = filename
    data['_relative_path'] = filename

    new_content = PythonPackageContent.objects.create(
        filename=filename,
        packagetype=metadata.packagetype,
        name=data['classifiers'],
        version=data['version'])

    queryset = PythonPackageContent.objects.filter(pk=new_content.pk)

    if repository_pk:
        repository = Repository.objects.get(pk=repository_pk)
        with RepositoryVersion.create(repository) as new_version:
            new_version.add_content(queryset)

    resource = CreatedResource(content_object=new_content)
    resource.save()
Esempio n. 12
0
def create_content(artifact_sha256, filename):
    """
    Creates PythonPackageContent from artifact.

    Args:
        artifact_sha256: validated artifact
        filename: file name
    Returns:
        queryset of the new created content
    """
    # iterate through extensions since splitext does not support things like .tar.gz
    extensions = list(DIST_EXTENSIONS.keys())
    pkg_type_index = [filename.endswith(ext) for ext in extensions].index(True)
    packagetype = DIST_EXTENSIONS[extensions[pkg_type_index]]
    # Copy file to a temp directory under the user provided filename, we do this
    # because pkginfo validates that the filename has a valid extension before
    # reading it
    artifact = Artifact.objects.get(sha256=artifact_sha256)
    artifact_file = storage.open(artifact.file.name)
    with tempfile.NamedTemporaryFile('wb', suffix=filename) as temp_file:
        shutil.copyfileobj(artifact_file, temp_file)
        temp_file.flush()
        metadata = DIST_TYPES[packagetype](temp_file.name)
        metadata.packagetype = packagetype

    data = parse_project_metadata(vars(metadata))
    data['packagetype'] = metadata.packagetype
    data['version'] = metadata.version
    data['filename'] = filename
    data['sha256'] = artifact.sha256

    @transaction.atomic()
    def create():
        content = PythonPackageContent.objects.create(**data)
        ContentArtifact.objects.create(artifact=artifact,
                                       content=content,
                                       relative_path=filename)
        return content

    new_content = create()
    resource = CreatedResource(content_object=new_content)
    resource.save()

    return PythonPackageContent.objects.filter(pk=new_content.pk)
Esempio n. 13
0
def one_shot_upload(artifact_pk, filename, repository_pk=None):
    """
    One shot upload for RPM package.

    Args:
        artifact_pk: validated artifact for a file
        filename : name of file
        repository_pk: repository to extend with new pkg
    """
    artifact = Artifact.objects.get(pk=artifact_pk)

    # export META from rpm and prepare dict as saveable format
    try:
        new_pkg = _prepare_package(artifact, filename)
    except OSError:
        raise OSError('RPM file cannot be parsed for metadata.')

    pkg, created = Package.objects.get_or_create(**new_pkg)

    if not created:
        raise OSError('RPM package {} already exists.'.format(pkg.filename))

    ContentArtifact.objects.create(
        artifact=artifact,
        content=pkg,
        relative_path=filename
    )

    resource = CreatedResource(content_object=pkg)
    resource.save()

    if repository_pk:
        repository = Repository.objects.get(pk=repository_pk)
        content_to_add = Package.objects.filter(pkgId=pkg.pkgId)

        # create new repo version with uploaded package
        with RepositoryVersion.create(repository) as new_version:
            new_version.add_content(content_to_add)
Esempio n. 14
0
def migrate_from_pulp2(migration_plan_pk, validate=False, dry_run=False):
    """
    Main task to migrate from Pulp 2 to Pulp 3.

    Schedule other tasks based on the specified Migration Plan.

    Args:
        migration_plan_pk (str): The migration plan PK.
        validate (bool): If True, don't migrate unless validation is successful.
        dry_run (bool): If True, nothing is migrated, only validation happens.
    """
    def get_repo_types(plan):
        """
        Create mappings for pulp 2 repository types.

        Identify type by inspecting content of a repo.
        One mapping is repo_id -> repo_type, the other is repo_type -> list of repo_ids.

        It's used later during pre-migration and identification of removed repos from pulp 2

        Args:
            plan(MigrationPlan): A Migration Plan

        Returns:
            repo_id_to_type(dict): mapping from a pulp 2 repo_id to a plugin/repo type
            type_to_repo_ids(dict): mapping from a plugin/repo type to the list of repo_ids

        """
        repo_id_to_type = {}
        type_to_repo_ids = defaultdict(set)

        # mapping content type -> plugin/repo type, e.g. 'docker_blob' -> 'docker'
        content_type_to_plugin = {}

        for plugin in plan.get_plugin_plans():
            for content_type in plugin.migrator.pulp2_content_models:
                content_type_to_plugin[
                    content_type] = plugin.migrator.pulp2_plugin

            repos = set(plugin.get_repositories())
            repos |= set(plugin.get_importers_repos())
            repos |= set(plugin.get_distributors_repos())

            for repo in repos:
                repo_id_to_type[repo] = plugin.type
            type_to_repo_ids[plugin.type].update(repos)

        # TODO: optimizations.
        # It looks at each content at the moment. Potential optimizations:
        #  - This is a big query, paginate?
        #  - Filter by repos from the plan
        #  - Query any but one record for a repo
        for rec in RepositoryContentUnit.objects().\
                only('repo_id', 'unit_type_id').as_pymongo().no_cache():
            repo_id = rec['repo_id']
            unit_type_id = rec['unit_type_id']

            # a type for a repo is already known or this content/repo type is not supported
            if repo_id in repo_id_to_type or unit_type_id not in content_type_to_plugin:
                continue
            plugin_name = content_type_to_plugin[unit_type_id]
            repo_id_to_type[repo_id] = plugin_name
            type_to_repo_ids[plugin_name].add(repo_id)

        return repo_id_to_type, type_to_repo_ids

    # MongoDB connection initialization
    connection.initialize()

    plan = MigrationPlan.objects.get(pk=migration_plan_pk)
    missing_resources = plan.get_missing_resources()

    if (validate or dry_run) and missing_resources:
        raise PlanValidationError(
            "Validation failed: resources missing {}".format(
                missing_resources))

    if dry_run:
        return

    task_group = TaskGroup(description="Migration Sub-tasks")
    task_group.save()
    GroupProgressReport(message="Repo version creation",
                        code="create.repo_version",
                        task_group=task_group).save()
    GroupProgressReport(message="Distribution creation",
                        code="create.distribution",
                        task_group=task_group).save()
    current_task = Task.current()
    current_task.task_group = task_group
    current_task.save()
    resource = CreatedResource(content_object=task_group)
    resource.save()

    # call it here and not inside steps below to generate mapping only once
    repo_id_to_type, type_to_repo_ids = get_repo_types(plan)

    # TODO: if plan is empty for a plugin, only migrate downloaded content

    pre_migrate_all_without_content(plan, type_to_repo_ids, repo_id_to_type)
    pre_migrate_all_content(plan)
    mark_removed_resources(plan, type_to_repo_ids)
    delete_old_resources(plan)
    migrate_repositories(plan)
    migrate_importers(plan)
    migrate_content(plan)
    create_repoversions_publications_distributions(plan)

    task_group.finish()