def sign(repository_pk, signing_service_pk, reference, tags_list=None):
"""
Create a new repository version by signing manifests.
Create signature for each manifest that is specified and add it to the repo.
If no manifests were specified, then sign all manifests in the repo.
What manifests to sign is identified by tag.
Manifest lists are not signed. Image manifests from manifest list are signed by digest.
Args:
repository_pk (uuid): A pk for a Repository for which a new Repository Version should be
created.
signing_service_pk (uuid): A pk of the signing service to use.
reference (str): Reference that will be used to produce signature.
tags_list (list): List of PKs for :class:`~pulp_container.app.models.Tag` manifests of which
should be signed.
"""
repository = Repository.objects.get(pk=repository_pk).cast()
latest_version = repository.latest_version()
if tags_list:
latest_repo_content_tags = latest_version.content.filter(
pulp_type=Tag.get_pulp_type(), pk__in=tags_list)
else:
latest_repo_content_tags = latest_version.content.filter(
pulp_type=Tag.get_pulp_type())
latest_repo_tags = Tag.objects.filter(pk__in=latest_repo_content_tags)
signing_service = ManifestSigningService.objects.get(pk=signing_service_pk)
added_signatures = []
already_signed = []
for tag in latest_repo_tags:
tagged_manifest = tag.tagged_manifest
if tagged_manifest.media_type in MANIFEST_MEDIA_TYPES.IMAGE:
signature_pk = create_signature(tagged_manifest, ":".join(
(reference, tag.name)), signing_service)
added_signatures.append(signature_pk)
elif tagged_manifest.media_type in MANIFEST_MEDIA_TYPES.LIST:
# parse ML and sign per-arches by digest
for manifest in tagged_manifest.listed_manifests.iterator():
# image manifests can be present in multiple ML within the repo
if manifest.digest not in already_signed:
signature_pk = create_signature(
manifest, ":".join((reference, manifest.digest)),
signing_service)
already_signed.append(manifest.digest)
added_signatures.append(signature_pk)
added_signatures_qs = ManifestSignature.objects.filter(
pk__in=added_signatures)
with repository.new_version() as new_version:
new_version.add_content(added_signatures_qs)
def create_tag(self, saved_artifact, url):
"""
Create `DeclarativeContent` for each tag.
Each dc contains enough information to be dowloaded by an ArtifactDownload Stage.
Args:
tag_name (str): Name of each tag
Returns:
pulpcore.plugin.stages.DeclarativeContent: A Tag DeclarativeContent object
"""
tag_name = url.split('/')[-1]
relative_url = '/v2/{name}/manifests/{tag}'.format(
name=self.remote.namespaced_upstream_name,
tag=tag_name,
)
url = urljoin(self.remote.url, relative_url)
tag = Tag(name=tag_name)
da = DeclarativeArtifact(
artifact=saved_artifact,
url=url,
relative_path=tag_name,
remote=self.remote,
extra_data={'headers': V2_ACCEPT_HEADERS}
)
tag_dc = DeclarativeContent(content=tag, d_artifacts=[da])
return tag_dc
def add_image_from_directory_to_repository(path, repository, tag):
"""
Creates a Manifest and all blobs from a directory with OCI image
Args:
path (str): Path to directory with the OCI image
repository (class:`pulpcore.plugin.models.Repository`): The destination repository
tag (str): Tag name for the new image in the repository
Returns:
A class:`pulpcore.plugin.models.RepositoryVersion` that contains the new OCI container
image and tag.
"""
manifest_path = "{}manifest.json".format(path)
manifest_artifact = Artifact.init_and_validate(manifest_path)
manifest_artifact.save()
manifest_digest = "sha256:{}".format(manifest_artifact.sha256)
manifest = Manifest(digest=manifest_digest,
schema_version=2,
media_type=MEDIA_TYPE.MANIFEST_OCI)
manifest.save()
ContentArtifact(artifact=manifest_artifact,
content=manifest,
relative_path=manifest_digest).save()
tag = Tag(name=tag, tagged_manifest=manifest)
tag.save()
ContentArtifact(artifact=manifest_artifact,
content=tag,
relative_path=tag.name).save()
with repository.new_version() as new_repo_version:
new_repo_version.add_content(Manifest.objects.filter(pk=manifest.pk))
new_repo_version.add_content(Tag.objects.filter(pk=tag.pk))
with open(manifest_artifact.file.path, "r") as manifest_file:
manifest_json = json.load(manifest_file)
config_blob = get_or_create_blob(manifest_json["config"], manifest,
path)
manifest.config_blob = config_blob
manifest.save()
new_repo_version.add_content(
Blob.objects.filter(pk=config_blob.pk))
for layer in manifest_json["layers"]:
blob = get_or_create_blob(layer, manifest, path)
new_repo_version.add_content(Blob.objects.filter(pk=blob.pk))
return new_repo_version
def untag_image(tag, repository_pk):
"""
Create a new repository version without a specified manifest's tag name.
"""
repository = Repository.objects.get(pk=repository_pk).cast()
latest_version = repository.latest_version()
tags_in_latest_repository = latest_version.content.filter(pulp_type=Tag.get_pulp_type())
tags_to_remove = Tag.objects.filter(pk__in=tags_in_latest_repository, name=tag)
with repository.new_version() as repository_version:
repository_version.remove_content(tags_to_remove)
def create_pulp3_content(self):
"""
Create a Pulp 3 Tag unit for saving it later in a bulk operation.
"""
return Tag(name=self.name)
async def run(self):
"""
ContainerFirstStage.
"""
future_manifests = []
tag_list = []
tag_dcs = []
to_download = []
man_dcs = {}
async with ProgressReport(message="Downloading tag list",
code="sync.downloading.tag_list",
total=1) as pb:
repo_name = self.remote.namespaced_upstream_name
relative_url = "/v2/{name}/tags/list".format(name=repo_name)
tag_list_url = urljoin(self.remote.url, relative_url)
list_downloader = self.remote.get_downloader(url=tag_list_url)
await list_downloader.run(extra_data={"repo_name": repo_name})
with open(list_downloader.path) as tags_raw:
tags_dict = json.loads(tags_raw.read())
tag_list = tags_dict["tags"]
# check for the presence of the pagination link header
link = list_downloader.response_headers.get("Link")
await self.handle_pagination(link, repo_name, tag_list)
tag_list = self.filter_tags(tag_list)
await pb.aincrement()
for tag_name in tag_list:
relative_url = "/v2/{name}/manifests/{tag}".format(
name=self.remote.namespaced_upstream_name, tag=tag_name)
url = urljoin(self.remote.url, relative_url)
downloader = self.remote.get_downloader(url=url)
to_download.append(
downloader.run(extra_data={"headers": V2_ACCEPT_HEADERS}))
async with ProgressReport(
message="Processing Tags",
code="sync.processing.tag",
state=TASK_STATES.RUNNING,
total=len(tag_list),
) as pb_parsed_tags:
for download_tag in asyncio.as_completed(to_download):
tag = await download_tag
with open(tag.path, "rb") as content_file:
raw_data = content_file.read()
tag.artifact_attributes["file"] = tag.path
saved_artifact = await sync_to_async(_save_artifact_blocking)(
tag.artifact_attributes)
tag_name = tag.url.split("/")[-1]
tag_dc = DeclarativeContent(Tag(name=tag_name))
content_data = json.loads(raw_data)
media_type = content_data.get("mediaType")
if media_type in (MEDIA_TYPE.MANIFEST_LIST,
MEDIA_TYPE.INDEX_OCI):
list_dc = self.create_tagged_manifest_list(
tag_name, saved_artifact, content_data)
await self.put(list_dc)
tag_dc.extra_data["tagged_manifest_dc"] = list_dc
for manifest_data in content_data.get("manifests"):
man_dc = self.create_manifest(list_dc, manifest_data)
future_manifests.append(man_dc)
man_dcs[man_dc.content.digest] = man_dc
await self.put(man_dc)
else:
man_dc = self.create_tagged_manifest(
tag_name, saved_artifact, content_data, raw_data)
await self.put(man_dc)
tag_dc.extra_data["tagged_manifest_dc"] = man_dc
await self.handle_blobs(man_dc, content_data)
tag_dcs.append(tag_dc)
await pb_parsed_tags.aincrement()
for manifest_future in future_manifests:
man = await manifest_future.resolution()
artifact = await sync_to_async(man._artifacts.get)()
with artifact.file.open() as content_file:
raw = content_file.read()
content_data = json.loads(raw)
man_dc = man_dcs[man.digest]
await self.handle_blobs(man_dc, content_data)
for tag_dc in tag_dcs:
tagged_manifest_dc = tag_dc.extra_data["tagged_manifest_dc"]
tag_dc.content.tagged_manifest = await tagged_manifest_dc.resolution(
)
await self.put(tag_dc)
def recursive_remove_content(repository_pk, content_units):
"""
Create a new repository version by recursively removing content.
For each unit that is specified, we also need to remove related content,
unless that content is also related to content that will remain in the
repository. For example, if a manifest-list is specified, we need to remove
all referenced manifests unless those manifests are referenced by a
manifest-list that will stay in the repository.
For each content type, we identify 3 categories:
1. must_remain: These content units are referenced by content units that will not be removed
2. to_remove: These content units are either explicity given by the user,
or they are referenced by the content explicity given, and they are not in must_remain.
3. to_remain: Content in the repo that is not in to_remove. This category
is used to determine must_remain of lower heirarchy content.
Args:
repository_pk (int): The primary key for a Repository for which a new Repository Version
should be created.
content_units (list): List of PKs for :class:`~pulpcore.app.models.Content` that
should be removed from the Repository.
"""
repository = Repository.objects.get(pk=repository_pk).cast()
latest_version = repository.latest_version()
latest_content = latest_version.content.all() if latest_version else Content.objects.none()
if "*" in content_units:
with repository.new_version() as new_version:
new_version.remove_content(latest_content)
else:
tags_in_repo = Q(pk__in=latest_content.filter(pulp_type=Tag.get_pulp_type()))
sigs_in_repo = Q(pk__in=latest_content.filter(pulp_type=ManifestSignature.get_pulp_type()))
manifests_in_repo = Q(pk__in=latest_content.filter(pulp_type=Manifest.get_pulp_type()))
user_provided_content = Q(pk__in=content_units)
type_manifest_list = Q(media_type__in=[MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.INDEX_OCI])
manifest_media_types = [
MEDIA_TYPE.MANIFEST_V1,
MEDIA_TYPE.MANIFEST_V2,
MEDIA_TYPE.MANIFEST_OCI,
]
type_manifest = Q(media_type__in=manifest_media_types)
blobs_in_repo = Q(pk__in=latest_content.filter(pulp_type=Blob.get_pulp_type()))
# Tags do not have must_remain because they are the highest level content.
tags_to_remove = Tag.objects.filter(user_provided_content & tags_in_repo)
tags_to_remain = Tag.objects.filter(tags_in_repo).exclude(pk__in=tags_to_remove)
tagged_manifests_must_remain = Q(
pk__in=tags_to_remain.values_list("tagged_manifest", flat=True)
)
tagged_manifests_to_remove = Q(
pk__in=tags_to_remove.values_list("tagged_manifest", flat=True)
)
manifest_lists_must_remain = Manifest.objects.filter(
manifests_in_repo & tagged_manifests_must_remain & type_manifest_list
)
manifest_lists_to_remove = (
Manifest.objects.filter(user_provided_content | tagged_manifests_to_remove)
.filter(type_manifest_list & manifests_in_repo)
.exclude(pk__in=manifest_lists_must_remain)
)
manifest_lists_to_remain = Manifest.objects.filter(
manifests_in_repo & type_manifest_list
).exclude(pk__in=manifest_lists_to_remove)
listed_manifests_must_remain = Q(
pk__in=manifest_lists_to_remain.values_list("listed_manifests", flat=True)
)
manifests_must_remain = Manifest.objects.filter(
tagged_manifests_must_remain | listed_manifests_must_remain
).filter(type_manifest & manifests_in_repo)
listed_manifests_to_remove = Q(
pk__in=manifest_lists_to_remove.values_list("listed_manifests", flat=True)
)
manifests_to_remove = (
Manifest.objects.filter(
user_provided_content | listed_manifests_to_remove | tagged_manifests_to_remove
)
.filter(type_manifest & manifests_in_repo)
.exclude(pk__in=manifests_must_remain)
)
manifests_to_remain = Manifest.objects.filter(manifests_in_repo & type_manifest).exclude(
pk__in=manifests_to_remove
)
listed_blobs_must_remain = Q(
pk__in=manifests_to_remain.values_list("blobs", flat=True)
) | Q(pk__in=manifests_to_remain.values_list("config_blob", flat=True))
listed_blobs_to_remove = Q(pk__in=manifests_to_remove.values_list("blobs", flat=True)) | Q(
pk__in=manifests_to_remove.values_list("config_blob", flat=True)
)
blobs_to_remove = (
Blob.objects.filter(user_provided_content | listed_blobs_to_remove)
.filter(blobs_in_repo)
.exclude(listed_blobs_must_remain)
)
# signatures can't be shared, so no need to calculate which ones to remain
sigs_to_remove_from_manifests = Q(signed_manifest__in=manifests_to_remove)
signatures_to_remove = ManifestSignature.objects.filter(
(user_provided_content & sigs_in_repo) | sigs_to_remove_from_manifests
)
with repository.new_version() as new_version:
new_version.remove_content(tags_to_remove)
new_version.remove_content(manifest_lists_to_remove)
new_version.remove_content(manifests_to_remove)
new_version.remove_content(blobs_to_remove)
new_version.remove_content(signatures_to_remove)
async def run(self):
"""
ContainerFirstStage.
"""
async def get_signature_source(self):
"""
Find out where signatures come from: sigstore, extension API or not available at all.
"""
if self.remote.sigstore:
return SIGNATURE_SOURCE.SIGSTORE
registry_v2_url = urljoin(self.remote.url, "v2")
extension_check_downloader = self.remote.get_noauth_downloader(
url=registry_v2_url)
response_headers = {}
try:
result = await extension_check_downloader.run()
response_headers = result.headers
except aiohttp.client_exceptions.ClientResponseError as exc:
if exc.status == 401:
response_headers = dict(exc.headers)
if response_headers.get(SIGNATURE_HEADER) == "1":
return SIGNATURE_SOURCE.API_EXTENSION
tag_list = []
tag_dcs = []
to_download = []
man_dcs = {}
signature_dcs = []
signature_source = await get_signature_source(self)
if signature_source is None and self.signed_only:
raise ValueError(
_("It is requested to sync only signed content but no sigstore URL is "
"provided. Please configure a `sigstore` on your Remote or set "
"`signed_only` to `False` for your sync request."))
async with ProgressReport(message="Downloading tag list",
code="sync.downloading.tag_list",
total=1) as pb:
repo_name = self.remote.namespaced_upstream_name
relative_url = "/v2/{name}/tags/list".format(name=repo_name)
tag_list_url = urljoin(self.remote.url, relative_url)
list_downloader = self.remote.get_downloader(url=tag_list_url)
await list_downloader.run(extra_data={"repo_name": repo_name})
with open(list_downloader.path) as tags_raw:
tags_dict = json.loads(tags_raw.read())
tag_list = tags_dict["tags"]
# check for the presence of the pagination link header
link = list_downloader.response_headers.get("Link")
await self.handle_pagination(link, repo_name, tag_list)
tag_list = self.filter_tags(tag_list)
await pb.aincrement()
for tag_name in tag_list:
relative_url = "/v2/{name}/manifests/{tag}".format(
name=self.remote.namespaced_upstream_name, tag=tag_name)
url = urljoin(self.remote.url, relative_url)
downloader = self.remote.get_downloader(url=url)
to_download.append(
downloader.run(extra_data={"headers": V2_ACCEPT_HEADERS}))
async with ProgressReport(
message="Processing Tags",
code="sync.processing.tag",
state=TASK_STATES.RUNNING,
total=len(tag_list),
) as pb_parsed_tags:
for download_tag in asyncio.as_completed(to_download):
tag = await download_tag
with open(tag.path, "rb") as content_file:
raw_data = content_file.read()
tag.artifact_attributes["file"] = tag.path
saved_artifact = await sync_to_async(_save_artifact_blocking)(
tag.artifact_attributes)
tag_name = tag.url.split("/")[-1]
tag_dc = DeclarativeContent(Tag(name=tag_name))
content_data = json.loads(raw_data)
media_type = content_data.get("mediaType")
if media_type in (MEDIA_TYPE.MANIFEST_LIST,
MEDIA_TYPE.INDEX_OCI):
list_dc = self.create_tagged_manifest_list(
tag_name, saved_artifact, content_data)
for manifest_data in content_data.get("manifests"):
man_dc = self.create_manifest(list_dc, manifest_data)
if signature_source is not None:
man_sig_dcs = await self.create_signatures(
man_dc, signature_source)
if self.signed_only and not man_sig_dcs:
log.info(
_("The unsigned image {img_digest} which is a part of the "
"manifest list {ml_digest} (tagged as `{tag}`) can't be "
"synced due to a requirement to sync signed content only. "
"The whole manifest list is skipped.".
format(
img_digest=man_dc.d_content.digest,
ml_digest=list_dc.d_content.digest,
tag=tag_name,
)))
# do not pass down the pipeline a manifest list with unsigned
# manifests.
break
signature_dcs.extend(man_sig_dcs)
man_dcs[man_dc.content.digest] = man_dc
await self.put(man_dc)
else:
# only pass the manifest list and tag down the pipeline if there were no
# issues with signatures (no `break` in the `for` loop)
tag_dc.extra_data["tagged_manifest_dc"] = list_dc
await self.put(list_dc)
tag_dcs.append(tag_dc)
await pb_parsed_tags.aincrement()
else:
man_dc = self.create_tagged_manifest(
tag_name, saved_artifact, content_data, raw_data)
if signature_source is not None:
man_sig_dcs = await self.create_signatures(
man_dc, signature_source)
if self.signed_only and not man_sig_dcs:
# do not pass down the pipeline unsigned manifests
continue
signature_dcs.extend(man_sig_dcs)
await self.put(man_dc)
tag_dc.extra_data["tagged_manifest_dc"] = man_dc
await man_dc.resolution()
await self.handle_blobs(man_dc, content_data)
tag_dcs.append(tag_dc)
await pb_parsed_tags.aincrement()
for digest, man_dc in man_dcs.items():
man = await man_dc.resolution()
artifact = await sync_to_async(man._artifacts.get)()
with artifact.file.open() as content_file:
raw = content_file.read()
content_data = json.loads(raw)
await self.handle_blobs(man_dc, content_data)
for tag_dc in tag_dcs:
tagged_manifest_dc = tag_dc.extra_data["tagged_manifest_dc"]
tag_dc.content.tagged_manifest = await tagged_manifest_dc.resolution(
)
await self.put(tag_dc)
for sig_dc in signature_dcs:
signed_manifest_dc = sig_dc.extra_data["signed_manifest_dc"]
sig_dc.content.signed_manifest = await signed_manifest_dc.resolution(
)
await self.put(sig_dc)
def recursive_add_content(repository_pk, content_units):
"""
Create a new repository version by recursively adding content.
For each unit that is specified, we also need to add related content. For example, if a
manifest-list is specified, we need to add all referenced manifests, and all blobs referenced
by those manifests.
Args:
repository_pk (int): The primary key for a Repository for which a new Repository Version
should be created.
content_units (list): List of PKs for :class:`~pulpcore.app.models.Content` that
should be added to the previous Repository Version for this Repository.
"""
repository = ContainerRepository.objects.get(pk=repository_pk)
tags_to_add = Tag.objects.filter(pk__in=content_units)
manifest_lists_to_add = Manifest.objects.filter(
pk__in=content_units,
media_type__in=[
MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.INDEX_OCI
]) | Manifest.objects.filter(
pk__in=tags_to_add.values_list("tagged_manifest", flat=True),
media_type__in=[MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.INDEX_OCI],
)
manifests_to_add = (
Manifest.objects.filter(
pk__in=content_units,
media_type__in=[
MEDIA_TYPE.MANIFEST_V1,
MEDIA_TYPE.MANIFEST_V1_SIGNED,
MEDIA_TYPE.MANIFEST_V2,
MEDIA_TYPE.MANIFEST_OCI,
],
)
| Manifest.objects.filter(pk__in=manifest_lists_to_add.values_list(
"listed_manifests", flat=True))
| Manifest.objects.filter(
pk__in=tags_to_add.values_list("tagged_manifest", flat=True),
media_type__in=[
MEDIA_TYPE.MANIFEST_V1,
MEDIA_TYPE.MANIFEST_V1_SIGNED,
MEDIA_TYPE.MANIFEST_V2,
MEDIA_TYPE.MANIFEST_OCI,
],
))
blobs_to_add = (
Blob.objects.filter(pk__in=content_units)
| Blob.objects.filter(
pk__in=manifests_to_add.values_list("blobs", flat=True))
| Blob.objects.filter(
pk__in=manifests_to_add.values_list("config_blob", flat=True)))
latest_version = repository.latest_version()
if latest_version:
tags_in_repo = latest_version.content.filter(
pulp_type=Tag.get_pulp_type())
tags_to_replace = Tag.objects.filter(pk__in=tags_in_repo,
name__in=tags_to_add.values_list(
"name", flat=True))
else:
tags_to_replace = []
with repository.new_version() as new_version:
new_version.remove_content(tags_to_replace)
new_version.add_content(tags_to_add)
new_version.add_content(manifest_lists_to_add)
new_version.add_content(manifests_to_add)
new_version.add_content(blobs_to_add)
def create_pulp3_content(self):
"""
Create a Pulp 3 Tag unit for saving it later in a bulk operation.
"""
future_relations = {'tag_rel': self.tagged_manifest}
return (Tag(name=self.name), future_relations)
