def get_or_create_blob(layer_json, manifest, path): """ Creates Blob from json snippet of manifest.json Args: layer_json (json): json manifest (class:`pulp_container.app.models.Manifest`): The manifest path (str): Path of the directory that contains layer Returns: class:`pulp_container.app.models.Blob` """ try: blob = Blob.objects.get(digest=layer_json["digest"]) except Blob.DoesNotExist: layer_file_name = "{}{}".format(path, layer_json["digest"][7:]) layer_artifact = Artifact.init_and_validate(layer_file_name) layer_artifact.save() blob = Blob(digest=layer_json["digest"], media_type=layer_json["mediaType"]) blob.save() ContentArtifact(artifact=layer_artifact, content=blob, relative_path=layer_json["digest"]).save() if blob.media_type != MEDIA_TYPE.CONFIG_BLOB_OCI: BlobManifest(manifest=manifest, manifest_blob=blob).save() return blob
def create_blob(self, man_dc, blob_data): """ Create blob. Args: man_dc (pulpcore.plugin.stages.DeclarativeContent): dc for a ImageManifest blob_data (dict): Data about a blob """ digest = blob_data.get('digest') or blob_data.get('blobSum') blob_artifact = Artifact(sha256=digest[len("sha256:"):]) blob = Blob( digest=digest, media_type=blob_data.get('mediaType', MEDIA_TYPE.REGULAR_BLOB), ) relative_url = '/v2/{name}/blobs/{digest}'.format( name=self.remote.namespaced_upstream_name, digest=digest, ) blob_url = urljoin(self.remote.url, relative_url) da = DeclarativeArtifact( artifact=blob_artifact, url=blob_url, relative_path=digest, remote=self.remote, extra_data={'headers': V2_ACCEPT_HEADERS}, deferred_download=self.deferred_download ) blob_dc = DeclarativeContent( content=blob, d_artifacts=[da], ) return blob_dc
def create_blob(self, blob_data, deferred_download=True): """ Create blob. Args: blob_data (dict): Data about a blob deferred_download (bool): boolean that indicates whether not to download a blob immediatly. Config blob is downloaded regardless of the remote's settings """ digest = blob_data.get("digest") or blob_data.get("blobSum") blob_artifact = Artifact(sha256=digest[len("sha256:"):]) blob = Blob(digest=digest, media_type=blob_data.get("mediaType", MEDIA_TYPE.REGULAR_BLOB)) relative_url = "/v2/{name}/blobs/{digest}".format( name=self.remote.namespaced_upstream_name, digest=digest) blob_url = urljoin(self.remote.url, relative_url) da = DeclarativeArtifact( artifact=blob_artifact, url=blob_url, relative_path=digest, remote=self.remote, deferred_download=deferred_download and self.deferred_download, ) blob_dc = DeclarativeContent(content=blob, d_artifacts=[da]) return blob_dc
def create_blob(self, man_dc, blob_data): """ Create blob. Args: man_dc (pulpcore.plugin.stages.DeclarativeContent): dc for a ImageManifest blob_data (dict): Data about a blob """ digest = blob_data.get("digest") or blob_data.get("blobSum") blob_artifact = Artifact(sha256=digest[len("sha256:"):]) blob = Blob(digest=digest, media_type=blob_data.get("mediaType", MEDIA_TYPE.REGULAR_BLOB)) relative_url = "/v2/{name}/blobs/{digest}".format( name=self.remote.namespaced_upstream_name, digest=digest) blob_url = urljoin(self.remote.url, relative_url) da = DeclarativeArtifact( artifact=blob_artifact, url=blob_url, relative_path=digest, remote=self.remote, deferred_download=self.deferred_download, ) blob_dc = DeclarativeContent(content=blob, d_artifacts=[da]) return blob_dc
def create_pulp3_content(self): """ Create a Pulp 3 Blob unit for saving it later in a bulk operation. """ return Blob(digest=self.digest, media_type=self.media_type)
def recursive_remove_content(repository_pk, content_units): """ Create a new repository version by recursively removing content. For each unit that is specified, we also need to remove related content, unless that content is also related to content that will remain in the repository. For example, if a manifest-list is specified, we need to remove all referenced manifests unless those manifests are referenced by a manifest-list that will stay in the repository. For each content type, we identify 3 categories: 1. must_remain: These content units are referenced by content units that will not be removed 2. to_remove: These content units are either explicity given by the user, or they are referenced by the content explicity given, and they are not in must_remain. 3. to_remain: Content in the repo that is not in to_remove. This category is used to determine must_remain of lower heirarchy content. Args: repository_pk (int): The primary key for a Repository for which a new Repository Version should be created. content_units (list): List of PKs for :class:`~pulpcore.app.models.Content` that should be removed from the Repository. """ repository = Repository.objects.get(pk=repository_pk).cast() latest_version = repository.latest_version() latest_content = latest_version.content.all() if latest_version else Content.objects.none() if "*" in content_units: with repository.new_version() as new_version: new_version.remove_content(latest_content) else: tags_in_repo = Q(pk__in=latest_content.filter(pulp_type=Tag.get_pulp_type())) sigs_in_repo = Q(pk__in=latest_content.filter(pulp_type=ManifestSignature.get_pulp_type())) manifests_in_repo = Q(pk__in=latest_content.filter(pulp_type=Manifest.get_pulp_type())) user_provided_content = Q(pk__in=content_units) type_manifest_list = Q(media_type__in=[MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.INDEX_OCI]) manifest_media_types = [ MEDIA_TYPE.MANIFEST_V1, MEDIA_TYPE.MANIFEST_V2, MEDIA_TYPE.MANIFEST_OCI, ] type_manifest = Q(media_type__in=manifest_media_types) blobs_in_repo = Q(pk__in=latest_content.filter(pulp_type=Blob.get_pulp_type())) # Tags do not have must_remain because they are the highest level content. tags_to_remove = Tag.objects.filter(user_provided_content & tags_in_repo) tags_to_remain = Tag.objects.filter(tags_in_repo).exclude(pk__in=tags_to_remove) tagged_manifests_must_remain = Q( pk__in=tags_to_remain.values_list("tagged_manifest", flat=True) ) tagged_manifests_to_remove = Q( pk__in=tags_to_remove.values_list("tagged_manifest", flat=True) ) manifest_lists_must_remain = Manifest.objects.filter( manifests_in_repo & tagged_manifests_must_remain & type_manifest_list ) manifest_lists_to_remove = ( Manifest.objects.filter(user_provided_content | tagged_manifests_to_remove) .filter(type_manifest_list & manifests_in_repo) .exclude(pk__in=manifest_lists_must_remain) ) manifest_lists_to_remain = Manifest.objects.filter( manifests_in_repo & type_manifest_list ).exclude(pk__in=manifest_lists_to_remove) listed_manifests_must_remain = Q( pk__in=manifest_lists_to_remain.values_list("listed_manifests", flat=True) ) manifests_must_remain = Manifest.objects.filter( tagged_manifests_must_remain | listed_manifests_must_remain ).filter(type_manifest & manifests_in_repo) listed_manifests_to_remove = Q( pk__in=manifest_lists_to_remove.values_list("listed_manifests", flat=True) ) manifests_to_remove = ( Manifest.objects.filter( user_provided_content | listed_manifests_to_remove | tagged_manifests_to_remove ) .filter(type_manifest & manifests_in_repo) .exclude(pk__in=manifests_must_remain) ) manifests_to_remain = Manifest.objects.filter(manifests_in_repo & type_manifest).exclude( pk__in=manifests_to_remove ) listed_blobs_must_remain = Q( pk__in=manifests_to_remain.values_list("blobs", flat=True) ) | Q(pk__in=manifests_to_remain.values_list("config_blob", flat=True)) listed_blobs_to_remove = Q(pk__in=manifests_to_remove.values_list("blobs", flat=True)) | Q( pk__in=manifests_to_remove.values_list("config_blob", flat=True) ) blobs_to_remove = ( Blob.objects.filter(user_provided_content | listed_blobs_to_remove) .filter(blobs_in_repo) .exclude(listed_blobs_must_remain) ) # signatures can't be shared, so no need to calculate which ones to remain sigs_to_remove_from_manifests = Q(signed_manifest__in=manifests_to_remove) signatures_to_remove = ManifestSignature.objects.filter( (user_provided_content & sigs_in_repo) | sigs_to_remove_from_manifests ) with repository.new_version() as new_version: new_version.remove_content(tags_to_remove) new_version.remove_content(manifest_lists_to_remove) new_version.remove_content(manifests_to_remove) new_version.remove_content(blobs_to_remove) new_version.remove_content(signatures_to_remove)