def sign(repository_pk, signing_service_pk, reference, tags_list=None): """ Create a new repository version by signing manifests. Create signature for each manifest that is specified and add it to the repo. If no manifests were specified, then sign all manifests in the repo. What manifests to sign is identified by tag. Manifest lists are not signed. Image manifests from manifest list are signed by digest. Args: repository_pk (uuid): A pk for a Repository for which a new Repository Version should be created. signing_service_pk (uuid): A pk of the signing service to use. reference (str): Reference that will be used to produce signature. tags_list (list): List of PKs for :class:`~pulp_container.app.models.Tag` manifests of which should be signed. """ repository = Repository.objects.get(pk=repository_pk).cast() latest_version = repository.latest_version() if tags_list: latest_repo_content_tags = latest_version.content.filter( pulp_type=Tag.get_pulp_type(), pk__in=tags_list) else: latest_repo_content_tags = latest_version.content.filter( pulp_type=Tag.get_pulp_type()) latest_repo_tags = Tag.objects.filter(pk__in=latest_repo_content_tags) signing_service = ManifestSigningService.objects.get(pk=signing_service_pk) added_signatures = [] already_signed = [] for tag in latest_repo_tags: tagged_manifest = tag.tagged_manifest if tagged_manifest.media_type in MANIFEST_MEDIA_TYPES.IMAGE: signature_pk = create_signature(tagged_manifest, ":".join( (reference, tag.name)), signing_service) added_signatures.append(signature_pk) elif tagged_manifest.media_type in MANIFEST_MEDIA_TYPES.LIST: # parse ML and sign per-arches by digest for manifest in tagged_manifest.listed_manifests.iterator(): # image manifests can be present in multiple ML within the repo if manifest.digest not in already_signed: signature_pk = create_signature( manifest, ":".join((reference, manifest.digest)), signing_service) already_signed.append(manifest.digest) added_signatures.append(signature_pk) added_signatures_qs = ManifestSignature.objects.filter( pk__in=added_signatures) with repository.new_version() as new_version: new_version.add_content(added_signatures_qs)
def create_tag(self, saved_artifact, url): """ Create `DeclarativeContent` for each tag. Each dc contains enough information to be dowloaded by an ArtifactDownload Stage. Args: tag_name (str): Name of each tag Returns: pulpcore.plugin.stages.DeclarativeContent: A Tag DeclarativeContent object """ tag_name = url.split('/')[-1] relative_url = '/v2/{name}/manifests/{tag}'.format( name=self.remote.namespaced_upstream_name, tag=tag_name, ) url = urljoin(self.remote.url, relative_url) tag = Tag(name=tag_name) da = DeclarativeArtifact( artifact=saved_artifact, url=url, relative_path=tag_name, remote=self.remote, extra_data={'headers': V2_ACCEPT_HEADERS} ) tag_dc = DeclarativeContent(content=tag, d_artifacts=[da]) return tag_dc
def add_image_from_directory_to_repository(path, repository, tag): """ Creates a Manifest and all blobs from a directory with OCI image Args: path (str): Path to directory with the OCI image repository (class:`pulpcore.plugin.models.Repository`): The destination repository tag (str): Tag name for the new image in the repository Returns: A class:`pulpcore.plugin.models.RepositoryVersion` that contains the new OCI container image and tag. """ manifest_path = "{}manifest.json".format(path) manifest_artifact = Artifact.init_and_validate(manifest_path) manifest_artifact.save() manifest_digest = "sha256:{}".format(manifest_artifact.sha256) manifest = Manifest(digest=manifest_digest, schema_version=2, media_type=MEDIA_TYPE.MANIFEST_OCI) manifest.save() ContentArtifact(artifact=manifest_artifact, content=manifest, relative_path=manifest_digest).save() tag = Tag(name=tag, tagged_manifest=manifest) tag.save() ContentArtifact(artifact=manifest_artifact, content=tag, relative_path=tag.name).save() with repository.new_version() as new_repo_version: new_repo_version.add_content(Manifest.objects.filter(pk=manifest.pk)) new_repo_version.add_content(Tag.objects.filter(pk=tag.pk)) with open(manifest_artifact.file.path, "r") as manifest_file: manifest_json = json.load(manifest_file) config_blob = get_or_create_blob(manifest_json["config"], manifest, path) manifest.config_blob = config_blob manifest.save() new_repo_version.add_content( Blob.objects.filter(pk=config_blob.pk)) for layer in manifest_json["layers"]: blob = get_or_create_blob(layer, manifest, path) new_repo_version.add_content(Blob.objects.filter(pk=blob.pk)) return new_repo_version
def untag_image(tag, repository_pk): """ Create a new repository version without a specified manifest's tag name. """ repository = Repository.objects.get(pk=repository_pk).cast() latest_version = repository.latest_version() tags_in_latest_repository = latest_version.content.filter(pulp_type=Tag.get_pulp_type()) tags_to_remove = Tag.objects.filter(pk__in=tags_in_latest_repository, name=tag) with repository.new_version() as repository_version: repository_version.remove_content(tags_to_remove)
def create_pulp3_content(self): """ Create a Pulp 3 Tag unit for saving it later in a bulk operation. """ return Tag(name=self.name)
async def run(self): """ ContainerFirstStage. """ future_manifests = [] tag_list = [] tag_dcs = [] to_download = [] man_dcs = {} async with ProgressReport(message="Downloading tag list", code="sync.downloading.tag_list", total=1) as pb: repo_name = self.remote.namespaced_upstream_name relative_url = "/v2/{name}/tags/list".format(name=repo_name) tag_list_url = urljoin(self.remote.url, relative_url) list_downloader = self.remote.get_downloader(url=tag_list_url) await list_downloader.run(extra_data={"repo_name": repo_name}) with open(list_downloader.path) as tags_raw: tags_dict = json.loads(tags_raw.read()) tag_list = tags_dict["tags"] # check for the presence of the pagination link header link = list_downloader.response_headers.get("Link") await self.handle_pagination(link, repo_name, tag_list) tag_list = self.filter_tags(tag_list) await pb.aincrement() for tag_name in tag_list: relative_url = "/v2/{name}/manifests/{tag}".format( name=self.remote.namespaced_upstream_name, tag=tag_name) url = urljoin(self.remote.url, relative_url) downloader = self.remote.get_downloader(url=url) to_download.append( downloader.run(extra_data={"headers": V2_ACCEPT_HEADERS})) async with ProgressReport( message="Processing Tags", code="sync.processing.tag", state=TASK_STATES.RUNNING, total=len(tag_list), ) as pb_parsed_tags: for download_tag in asyncio.as_completed(to_download): tag = await download_tag with open(tag.path, "rb") as content_file: raw_data = content_file.read() tag.artifact_attributes["file"] = tag.path saved_artifact = await sync_to_async(_save_artifact_blocking)( tag.artifact_attributes) tag_name = tag.url.split("/")[-1] tag_dc = DeclarativeContent(Tag(name=tag_name)) content_data = json.loads(raw_data) media_type = content_data.get("mediaType") if media_type in (MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.INDEX_OCI): list_dc = self.create_tagged_manifest_list( tag_name, saved_artifact, content_data) await self.put(list_dc) tag_dc.extra_data["tagged_manifest_dc"] = list_dc for manifest_data in content_data.get("manifests"): man_dc = self.create_manifest(list_dc, manifest_data) future_manifests.append(man_dc) man_dcs[man_dc.content.digest] = man_dc await self.put(man_dc) else: man_dc = self.create_tagged_manifest( tag_name, saved_artifact, content_data, raw_data) await self.put(man_dc) tag_dc.extra_data["tagged_manifest_dc"] = man_dc await self.handle_blobs(man_dc, content_data) tag_dcs.append(tag_dc) await pb_parsed_tags.aincrement() for manifest_future in future_manifests: man = await manifest_future.resolution() artifact = await sync_to_async(man._artifacts.get)() with artifact.file.open() as content_file: raw = content_file.read() content_data = json.loads(raw) man_dc = man_dcs[man.digest] await self.handle_blobs(man_dc, content_data) for tag_dc in tag_dcs: tagged_manifest_dc = tag_dc.extra_data["tagged_manifest_dc"] tag_dc.content.tagged_manifest = await tagged_manifest_dc.resolution( ) await self.put(tag_dc)
def recursive_remove_content(repository_pk, content_units): """ Create a new repository version by recursively removing content. For each unit that is specified, we also need to remove related content, unless that content is also related to content that will remain in the repository. For example, if a manifest-list is specified, we need to remove all referenced manifests unless those manifests are referenced by a manifest-list that will stay in the repository. For each content type, we identify 3 categories: 1. must_remain: These content units are referenced by content units that will not be removed 2. to_remove: These content units are either explicity given by the user, or they are referenced by the content explicity given, and they are not in must_remain. 3. to_remain: Content in the repo that is not in to_remove. This category is used to determine must_remain of lower heirarchy content. Args: repository_pk (int): The primary key for a Repository for which a new Repository Version should be created. content_units (list): List of PKs for :class:`~pulpcore.app.models.Content` that should be removed from the Repository. """ repository = Repository.objects.get(pk=repository_pk).cast() latest_version = repository.latest_version() latest_content = latest_version.content.all() if latest_version else Content.objects.none() if "*" in content_units: with repository.new_version() as new_version: new_version.remove_content(latest_content) else: tags_in_repo = Q(pk__in=latest_content.filter(pulp_type=Tag.get_pulp_type())) sigs_in_repo = Q(pk__in=latest_content.filter(pulp_type=ManifestSignature.get_pulp_type())) manifests_in_repo = Q(pk__in=latest_content.filter(pulp_type=Manifest.get_pulp_type())) user_provided_content = Q(pk__in=content_units) type_manifest_list = Q(media_type__in=[MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.INDEX_OCI]) manifest_media_types = [ MEDIA_TYPE.MANIFEST_V1, MEDIA_TYPE.MANIFEST_V2, MEDIA_TYPE.MANIFEST_OCI, ] type_manifest = Q(media_type__in=manifest_media_types) blobs_in_repo = Q(pk__in=latest_content.filter(pulp_type=Blob.get_pulp_type())) # Tags do not have must_remain because they are the highest level content. tags_to_remove = Tag.objects.filter(user_provided_content & tags_in_repo) tags_to_remain = Tag.objects.filter(tags_in_repo).exclude(pk__in=tags_to_remove) tagged_manifests_must_remain = Q( pk__in=tags_to_remain.values_list("tagged_manifest", flat=True) ) tagged_manifests_to_remove = Q( pk__in=tags_to_remove.values_list("tagged_manifest", flat=True) ) manifest_lists_must_remain = Manifest.objects.filter( manifests_in_repo & tagged_manifests_must_remain & type_manifest_list ) manifest_lists_to_remove = ( Manifest.objects.filter(user_provided_content | tagged_manifests_to_remove) .filter(type_manifest_list & manifests_in_repo) .exclude(pk__in=manifest_lists_must_remain) ) manifest_lists_to_remain = Manifest.objects.filter( manifests_in_repo & type_manifest_list ).exclude(pk__in=manifest_lists_to_remove) listed_manifests_must_remain = Q( pk__in=manifest_lists_to_remain.values_list("listed_manifests", flat=True) ) manifests_must_remain = Manifest.objects.filter( tagged_manifests_must_remain | listed_manifests_must_remain ).filter(type_manifest & manifests_in_repo) listed_manifests_to_remove = Q( pk__in=manifest_lists_to_remove.values_list("listed_manifests", flat=True) ) manifests_to_remove = ( Manifest.objects.filter( user_provided_content | listed_manifests_to_remove | tagged_manifests_to_remove ) .filter(type_manifest & manifests_in_repo) .exclude(pk__in=manifests_must_remain) ) manifests_to_remain = Manifest.objects.filter(manifests_in_repo & type_manifest).exclude( pk__in=manifests_to_remove ) listed_blobs_must_remain = Q( pk__in=manifests_to_remain.values_list("blobs", flat=True) ) | Q(pk__in=manifests_to_remain.values_list("config_blob", flat=True)) listed_blobs_to_remove = Q(pk__in=manifests_to_remove.values_list("blobs", flat=True)) | Q( pk__in=manifests_to_remove.values_list("config_blob", flat=True) ) blobs_to_remove = ( Blob.objects.filter(user_provided_content | listed_blobs_to_remove) .filter(blobs_in_repo) .exclude(listed_blobs_must_remain) ) # signatures can't be shared, so no need to calculate which ones to remain sigs_to_remove_from_manifests = Q(signed_manifest__in=manifests_to_remove) signatures_to_remove = ManifestSignature.objects.filter( (user_provided_content & sigs_in_repo) | sigs_to_remove_from_manifests ) with repository.new_version() as new_version: new_version.remove_content(tags_to_remove) new_version.remove_content(manifest_lists_to_remove) new_version.remove_content(manifests_to_remove) new_version.remove_content(blobs_to_remove) new_version.remove_content(signatures_to_remove)
async def run(self): """ ContainerFirstStage. """ async def get_signature_source(self): """ Find out where signatures come from: sigstore, extension API or not available at all. """ if self.remote.sigstore: return SIGNATURE_SOURCE.SIGSTORE registry_v2_url = urljoin(self.remote.url, "v2") extension_check_downloader = self.remote.get_noauth_downloader( url=registry_v2_url) response_headers = {} try: result = await extension_check_downloader.run() response_headers = result.headers except aiohttp.client_exceptions.ClientResponseError as exc: if exc.status == 401: response_headers = dict(exc.headers) if response_headers.get(SIGNATURE_HEADER) == "1": return SIGNATURE_SOURCE.API_EXTENSION tag_list = [] tag_dcs = [] to_download = [] man_dcs = {} signature_dcs = [] signature_source = await get_signature_source(self) if signature_source is None and self.signed_only: raise ValueError( _("It is requested to sync only signed content but no sigstore URL is " "provided. Please configure a `sigstore` on your Remote or set " "`signed_only` to `False` for your sync request.")) async with ProgressReport(message="Downloading tag list", code="sync.downloading.tag_list", total=1) as pb: repo_name = self.remote.namespaced_upstream_name relative_url = "/v2/{name}/tags/list".format(name=repo_name) tag_list_url = urljoin(self.remote.url, relative_url) list_downloader = self.remote.get_downloader(url=tag_list_url) await list_downloader.run(extra_data={"repo_name": repo_name}) with open(list_downloader.path) as tags_raw: tags_dict = json.loads(tags_raw.read()) tag_list = tags_dict["tags"] # check for the presence of the pagination link header link = list_downloader.response_headers.get("Link") await self.handle_pagination(link, repo_name, tag_list) tag_list = self.filter_tags(tag_list) await pb.aincrement() for tag_name in tag_list: relative_url = "/v2/{name}/manifests/{tag}".format( name=self.remote.namespaced_upstream_name, tag=tag_name) url = urljoin(self.remote.url, relative_url) downloader = self.remote.get_downloader(url=url) to_download.append( downloader.run(extra_data={"headers": V2_ACCEPT_HEADERS})) async with ProgressReport( message="Processing Tags", code="sync.processing.tag", state=TASK_STATES.RUNNING, total=len(tag_list), ) as pb_parsed_tags: for download_tag in asyncio.as_completed(to_download): tag = await download_tag with open(tag.path, "rb") as content_file: raw_data = content_file.read() tag.artifact_attributes["file"] = tag.path saved_artifact = await sync_to_async(_save_artifact_blocking)( tag.artifact_attributes) tag_name = tag.url.split("/")[-1] tag_dc = DeclarativeContent(Tag(name=tag_name)) content_data = json.loads(raw_data) media_type = content_data.get("mediaType") if media_type in (MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.INDEX_OCI): list_dc = self.create_tagged_manifest_list( tag_name, saved_artifact, content_data) for manifest_data in content_data.get("manifests"): man_dc = self.create_manifest(list_dc, manifest_data) if signature_source is not None: man_sig_dcs = await self.create_signatures( man_dc, signature_source) if self.signed_only and not man_sig_dcs: log.info( _("The unsigned image {img_digest} which is a part of the " "manifest list {ml_digest} (tagged as `{tag}`) can't be " "synced due to a requirement to sync signed content only. " "The whole manifest list is skipped.". format( img_digest=man_dc.d_content.digest, ml_digest=list_dc.d_content.digest, tag=tag_name, ))) # do not pass down the pipeline a manifest list with unsigned # manifests. break signature_dcs.extend(man_sig_dcs) man_dcs[man_dc.content.digest] = man_dc await self.put(man_dc) else: # only pass the manifest list and tag down the pipeline if there were no # issues with signatures (no `break` in the `for` loop) tag_dc.extra_data["tagged_manifest_dc"] = list_dc await self.put(list_dc) tag_dcs.append(tag_dc) await pb_parsed_tags.aincrement() else: man_dc = self.create_tagged_manifest( tag_name, saved_artifact, content_data, raw_data) if signature_source is not None: man_sig_dcs = await self.create_signatures( man_dc, signature_source) if self.signed_only and not man_sig_dcs: # do not pass down the pipeline unsigned manifests continue signature_dcs.extend(man_sig_dcs) await self.put(man_dc) tag_dc.extra_data["tagged_manifest_dc"] = man_dc await man_dc.resolution() await self.handle_blobs(man_dc, content_data) tag_dcs.append(tag_dc) await pb_parsed_tags.aincrement() for digest, man_dc in man_dcs.items(): man = await man_dc.resolution() artifact = await sync_to_async(man._artifacts.get)() with artifact.file.open() as content_file: raw = content_file.read() content_data = json.loads(raw) await self.handle_blobs(man_dc, content_data) for tag_dc in tag_dcs: tagged_manifest_dc = tag_dc.extra_data["tagged_manifest_dc"] tag_dc.content.tagged_manifest = await tagged_manifest_dc.resolution( ) await self.put(tag_dc) for sig_dc in signature_dcs: signed_manifest_dc = sig_dc.extra_data["signed_manifest_dc"] sig_dc.content.signed_manifest = await signed_manifest_dc.resolution( ) await self.put(sig_dc)
def recursive_add_content(repository_pk, content_units): """ Create a new repository version by recursively adding content. For each unit that is specified, we also need to add related content. For example, if a manifest-list is specified, we need to add all referenced manifests, and all blobs referenced by those manifests. Args: repository_pk (int): The primary key for a Repository for which a new Repository Version should be created. content_units (list): List of PKs for :class:`~pulpcore.app.models.Content` that should be added to the previous Repository Version for this Repository. """ repository = ContainerRepository.objects.get(pk=repository_pk) tags_to_add = Tag.objects.filter(pk__in=content_units) manifest_lists_to_add = Manifest.objects.filter( pk__in=content_units, media_type__in=[ MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.INDEX_OCI ]) | Manifest.objects.filter( pk__in=tags_to_add.values_list("tagged_manifest", flat=True), media_type__in=[MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.INDEX_OCI], ) manifests_to_add = ( Manifest.objects.filter( pk__in=content_units, media_type__in=[ MEDIA_TYPE.MANIFEST_V1, MEDIA_TYPE.MANIFEST_V1_SIGNED, MEDIA_TYPE.MANIFEST_V2, MEDIA_TYPE.MANIFEST_OCI, ], ) | Manifest.objects.filter(pk__in=manifest_lists_to_add.values_list( "listed_manifests", flat=True)) | Manifest.objects.filter( pk__in=tags_to_add.values_list("tagged_manifest", flat=True), media_type__in=[ MEDIA_TYPE.MANIFEST_V1, MEDIA_TYPE.MANIFEST_V1_SIGNED, MEDIA_TYPE.MANIFEST_V2, MEDIA_TYPE.MANIFEST_OCI, ], )) blobs_to_add = ( Blob.objects.filter(pk__in=content_units) | Blob.objects.filter( pk__in=manifests_to_add.values_list("blobs", flat=True)) | Blob.objects.filter( pk__in=manifests_to_add.values_list("config_blob", flat=True))) latest_version = repository.latest_version() if latest_version: tags_in_repo = latest_version.content.filter( pulp_type=Tag.get_pulp_type()) tags_to_replace = Tag.objects.filter(pk__in=tags_in_repo, name__in=tags_to_add.values_list( "name", flat=True)) else: tags_to_replace = [] with repository.new_version() as new_version: new_version.remove_content(tags_to_replace) new_version.add_content(tags_to_add) new_version.add_content(manifest_lists_to_add) new_version.add_content(manifests_to_add) new_version.add_content(blobs_to_add)
def create_pulp3_content(self): """ Create a Pulp 3 Tag unit for saving it later in a bulk operation. """ future_relations = {'tag_rel': self.tagged_manifest} return (Tag(name=self.name), future_relations)