def create_pulp3_content(self):
"""
Create a Pulp 3 Manifest unit for saving it later in a bulk operation.
"""
future_relations = {
"blob_rel": self.blobs,
"config_blob_rel": self.config_blob
}
return (
Manifest(
digest=self.digest,
media_type=self.media_type,
schema_version=self.schema_version,
),
future_relations,
)
def create_manifest(self, list_dc, manifest_data):
"""
Create an Image Manifest from manifest data in a ManifestList.
Args:
list_dc (pulpcore.plugin.stages.DeclarativeContent): dc for a ManifestList
manifest_data (dict): Data about a single new ImageManifest.
"""
digest = manifest_data["digest"]
relative_url = "/v2/{name}/manifests/{digest}".format(
name=self.remote.namespaced_upstream_name, digest=digest)
manifest_url = urljoin(self.remote.url, relative_url)
da = DeclarativeArtifact(
artifact=Artifact(),
url=manifest_url,
relative_path=digest,
remote=self.remote,
extra_data={"headers": V2_ACCEPT_HEADERS},
)
manifest = Manifest(
digest=manifest_data["digest"],
schema_version=2 if manifest_data["mediaType"]
in (MEDIA_TYPE.MANIFEST_V2, MEDIA_TYPE.MANIFEST_OCI) else 1,
media_type=manifest_data["mediaType"],
)
platform = {}
p = manifest_data["platform"]
platform["architecture"] = p["architecture"]
platform["os"] = p["os"]
platform["features"] = p.get("features", "")
platform["variant"] = p.get("variant", "")
platform["os.version"] = p.get("os.version", "")
platform["os.features"] = p.get("os.features", "")
man_dc = DeclarativeContent(
content=manifest,
d_artifacts=[da],
extra_data={
"relation": list_dc,
"platform": platform
},
)
return man_dc
def add_image_from_directory_to_repository(path, repository, tag):
"""
Creates a Manifest and all blobs from a directory with OCI image
Args:
path (str): Path to directory with the OCI image
repository (class:`pulpcore.plugin.models.Repository`): The destination repository
tag (str): Tag name for the new image in the repository
Returns:
A class:`pulpcore.plugin.models.RepositoryVersion` that contains the new OCI container
image and tag.
"""
manifest_path = "{}manifest.json".format(path)
manifest_artifact = Artifact.init_and_validate(manifest_path)
manifest_artifact.save()
manifest_digest = "sha256:{}".format(manifest_artifact.sha256)
manifest = Manifest(digest=manifest_digest,
schema_version=2,
media_type=MEDIA_TYPE.MANIFEST_OCI)
manifest.save()
ContentArtifact(artifact=manifest_artifact,
content=manifest,
relative_path=manifest_digest).save()
tag = Tag(name=tag, tagged_manifest=manifest)
tag.save()
ContentArtifact(artifact=manifest_artifact,
content=tag,
relative_path=tag.name).save()
with repository.new_version() as new_repo_version:
new_repo_version.add_content(Manifest.objects.filter(pk=manifest.pk))
new_repo_version.add_content(Tag.objects.filter(pk=tag.pk))
with open(manifest_artifact.file.path, "r") as manifest_file:
manifest_json = json.load(manifest_file)
config_blob = get_or_create_blob(manifest_json["config"], manifest,
path)
manifest.config_blob = config_blob
manifest.save()
new_repo_version.add_content(
Blob.objects.filter(pk=config_blob.pk))
for layer in manifest_json["layers"]:
blob = get_or_create_blob(layer, manifest, path)
new_repo_version.add_content(Blob.objects.filter(pk=blob.pk))
return new_repo_version
def create_manifest(self, list_dc, manifest_data):
"""
Create an Image Manifest from manifest data in a ManifestList.
Args:
list_dc (pulpcore.plugin.stages.DeclarativeContent): dc for a ManifestList
manifest_data (dict): Data about a single new ImageManifest.
"""
digest = manifest_data['digest']
relative_url = '/v2/{name}/manifests/{digest}'.format(
name=self.remote.namespaced_upstream_name,
digest=digest
)
manifest_url = urljoin(self.remote.url, relative_url)
da = DeclarativeArtifact(
artifact=Artifact(),
url=manifest_url,
relative_path=digest,
remote=self.remote,
extra_data={'headers': V2_ACCEPT_HEADERS}
)
manifest = Manifest(
digest=manifest_data['digest'],
schema_version=2 if manifest_data['mediaType'] in
(MEDIA_TYPE.MANIFEST_V2, MEDIA_TYPE.MANIFEST_OCI) else 1,
media_type=manifest_data['mediaType'],
)
platform = {}
p = manifest_data['platform']
platform['architecture'] = p['architecture']
platform['os'] = p['os']
platform['features'] = p.get('features', '')
platform['variant'] = p.get('variant', '')
platform['os.version'] = p.get('os.version', '')
platform['os.features'] = p.get('os.features', '')
man_dc = DeclarativeContent(
content=manifest,
d_artifacts=[da],
extra_data={'relation': list_dc, 'platform': platform},
)
return man_dc
def create_tagged_manifest_list(self, tag_name, saved_artifact,
manifest_list_data):
"""
Create a ManifestList.
Args:
tag_name (str): A name of a tag
saved_artifact (pulpcore.plugin.models.Artifact): A saved manifest's Artifact
manifest_list_data (dict): Data about a ManifestList
"""
digest = f"sha256:{saved_artifact.sha256}"
manifest_list = Manifest(
digest=digest,
schema_version=manifest_list_data["schemaVersion"],
media_type=manifest_list_data["mediaType"],
)
return self._create_manifest_declarative_content(
manifest_list, saved_artifact, tag_name, digest)
def create_tagged_manifest(self, tag_name, saved_artifact, manifest_data, raw_data):
"""
Create an Image Manifest.
Args:
tag_name (str): A name of a tag
saved_artifact (pulpcore.plugin.models.Artifact): A saved manifest's Artifact
manifest_data (dict): Data about a single new ImageManifest.
raw_data: (str): The raw JSON representation of the ImageManifest.
"""
media_type = manifest_data.get("mediaType", MEDIA_TYPE.MANIFEST_V1)
if media_type in (MEDIA_TYPE.MANIFEST_V2, MEDIA_TYPE.MANIFEST_OCI):
digest = f"sha256:{saved_artifact.sha256}"
else:
digest = self._calculate_digest(raw_data)
manifest = Manifest(
digest=digest, schema_version=manifest_data["schemaVersion"], media_type=media_type,
)
return self._create_manifest_declarative_content(manifest, saved_artifact, tag_name, digest)
def recursive_remove_content(repository_pk, content_units):
"""
Create a new repository version by recursively removing content.
For each unit that is specified, we also need to remove related content,
unless that content is also related to content that will remain in the
repository. For example, if a manifest-list is specified, we need to remove
all referenced manifests unless those manifests are referenced by a
manifest-list that will stay in the repository.
For each content type, we identify 3 categories:
1. must_remain: These content units are referenced by content units that will not be removed
2. to_remove: These content units are either explicity given by the user,
or they are referenced by the content explicity given, and they are not in must_remain.
3. to_remain: Content in the repo that is not in to_remove. This category
is used to determine must_remain of lower heirarchy content.
Args:
repository_pk (int): The primary key for a Repository for which a new Repository Version
should be created.
content_units (list): List of PKs for :class:`~pulpcore.app.models.Content` that
should be removed from the Repository.
"""
repository = Repository.objects.get(pk=repository_pk).cast()
latest_version = repository.latest_version()
latest_content = latest_version.content.all() if latest_version else Content.objects.none()
if "*" in content_units:
with repository.new_version() as new_version:
new_version.remove_content(latest_content)
else:
tags_in_repo = Q(pk__in=latest_content.filter(pulp_type=Tag.get_pulp_type()))
sigs_in_repo = Q(pk__in=latest_content.filter(pulp_type=ManifestSignature.get_pulp_type()))
manifests_in_repo = Q(pk__in=latest_content.filter(pulp_type=Manifest.get_pulp_type()))
user_provided_content = Q(pk__in=content_units)
type_manifest_list = Q(media_type__in=[MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.INDEX_OCI])
manifest_media_types = [
MEDIA_TYPE.MANIFEST_V1,
MEDIA_TYPE.MANIFEST_V2,
MEDIA_TYPE.MANIFEST_OCI,
]
type_manifest = Q(media_type__in=manifest_media_types)
blobs_in_repo = Q(pk__in=latest_content.filter(pulp_type=Blob.get_pulp_type()))
# Tags do not have must_remain because they are the highest level content.
tags_to_remove = Tag.objects.filter(user_provided_content & tags_in_repo)
tags_to_remain = Tag.objects.filter(tags_in_repo).exclude(pk__in=tags_to_remove)
tagged_manifests_must_remain = Q(
pk__in=tags_to_remain.values_list("tagged_manifest", flat=True)
)
tagged_manifests_to_remove = Q(
pk__in=tags_to_remove.values_list("tagged_manifest", flat=True)
)
manifest_lists_must_remain = Manifest.objects.filter(
manifests_in_repo & tagged_manifests_must_remain & type_manifest_list
)
manifest_lists_to_remove = (
Manifest.objects.filter(user_provided_content | tagged_manifests_to_remove)
.filter(type_manifest_list & manifests_in_repo)
.exclude(pk__in=manifest_lists_must_remain)
)
manifest_lists_to_remain = Manifest.objects.filter(
manifests_in_repo & type_manifest_list
).exclude(pk__in=manifest_lists_to_remove)
listed_manifests_must_remain = Q(
pk__in=manifest_lists_to_remain.values_list("listed_manifests", flat=True)
)
manifests_must_remain = Manifest.objects.filter(
tagged_manifests_must_remain | listed_manifests_must_remain
).filter(type_manifest & manifests_in_repo)
listed_manifests_to_remove = Q(
pk__in=manifest_lists_to_remove.values_list("listed_manifests", flat=True)
)
manifests_to_remove = (
Manifest.objects.filter(
user_provided_content | listed_manifests_to_remove | tagged_manifests_to_remove
)
.filter(type_manifest & manifests_in_repo)
.exclude(pk__in=manifests_must_remain)
)
manifests_to_remain = Manifest.objects.filter(manifests_in_repo & type_manifest).exclude(
pk__in=manifests_to_remove
)
listed_blobs_must_remain = Q(
pk__in=manifests_to_remain.values_list("blobs", flat=True)
) | Q(pk__in=manifests_to_remain.values_list("config_blob", flat=True))
listed_blobs_to_remove = Q(pk__in=manifests_to_remove.values_list("blobs", flat=True)) | Q(
pk__in=manifests_to_remove.values_list("config_blob", flat=True)
)
blobs_to_remove = (
Blob.objects.filter(user_provided_content | listed_blobs_to_remove)
.filter(blobs_in_repo)
.exclude(listed_blobs_must_remain)
)
# signatures can't be shared, so no need to calculate which ones to remain
sigs_to_remove_from_manifests = Q(signed_manifest__in=manifests_to_remove)
signatures_to_remove = ManifestSignature.objects.filter(
(user_provided_content & sigs_in_repo) | sigs_to_remove_from_manifests
)
with repository.new_version() as new_version:
new_version.remove_content(tags_to_remove)
new_version.remove_content(manifest_lists_to_remove)
new_version.remove_content(manifests_to_remove)
new_version.remove_content(blobs_to_remove)
new_version.remove_content(signatures_to_remove)