def get_parsed_manifest(self, validate=True): """ Returns the parsed manifest for this manifest. """ assert self.internal_manifest_bytes return parse_manifest_from_bytes(self.internal_manifest_bytes, self.media_type, validate=validate)
def _parse_manifest(): content_type = request.content_type or DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE if content_type == "application/json": # For back-compat. content_type = DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE try: return parse_manifest_from_bytes(Bytes.for_string_or_unicode(request.data), content_type) except ManifestException as me: logger.exception("failed to parse manifest when writing by tagname") raise ManifestInvalid(detail={"message": "failed to parse manifest: %s" % me})
def test_get_or_create_manifest_invalid_image(initialized_db): repository = get_repository("devtable", "simple") latest_tag = get_tag(repository, "latest") manifest_bytes = Bytes.for_string_or_unicode( latest_tag.manifest.manifest_bytes) parsed = parse_manifest_from_bytes(manifest_bytes, latest_tag.manifest.media_type.name, validate=False) builder = DockerSchema1ManifestBuilder("devtable", "simple", "anothertag") builder.add_layer(parsed.blob_digests[0], '{"id": "foo", "parent": "someinvalidimageid"}') sample_manifest_instance = builder.build(docker_v2_signing_key) created_manifest = get_or_create_manifest(repository, sample_manifest_instance, storage) assert created_manifest is None
def assert_gc_integrity(expect_storage_removed=True, check_oci_tags=True): """ Specialized assertion for ensuring that GC cleans up all dangling storages and labels, invokes the callback for images removed and doesn't invoke the callback for images *not* removed. """ # Add a callback for when images are removed. removed_image_storages = [] model.config.register_image_cleanup_callback(removed_image_storages.extend) # Store the number of dangling storages and labels. existing_storage_count = _get_dangling_storage_count() existing_label_count = _get_dangling_label_count() existing_manifest_count = _get_dangling_manifest_count() yield # Ensure the number of dangling storages, manifests and labels has not changed. updated_storage_count = _get_dangling_storage_count() assert updated_storage_count == existing_storage_count updated_label_count = _get_dangling_label_count() assert updated_label_count == existing_label_count, _get_dangling_labels() updated_manifest_count = _get_dangling_manifest_count() assert updated_manifest_count == existing_manifest_count # Ensure that for each call to the image+storage cleanup callback, the image and its # storage is not found *anywhere* in the database. for removed_image_and_storage in removed_image_storages: with pytest.raises(Image.DoesNotExist): Image.get(id=removed_image_and_storage.id) # Ensure that image storages are only removed if not shared. shared = Image.select().where( Image.storage == removed_image_and_storage.storage_id).count() if shared == 0: shared = (ManifestBlob.select().where( ManifestBlob.blob == removed_image_and_storage.storage_id).count()) if shared == 0: with pytest.raises(ImageStorage.DoesNotExist): ImageStorage.get(id=removed_image_and_storage.storage_id) with pytest.raises(ImageStorage.DoesNotExist): ImageStorage.get(uuid=removed_image_and_storage.storage.uuid) # Ensure all CAS storage is in the storage engine. preferred = storage.preferred_locations[0] for storage_row in ImageStorage.select(): if storage_row.cas_path: storage.get_content({preferred}, storage.blob_path( storage_row.content_checksum)) for blob_row in ApprBlob.select(): storage.get_content({preferred}, storage.blob_path(blob_row.digest)) # Ensure there are no danglings OCI tags. if check_oci_tags: oci_tags = {t.id for t in Tag.select()} referenced_oci_tags = {t.tag_id for t in TagToRepositoryTag.select()} assert not oci_tags - referenced_oci_tags # Ensure all tags have valid manifests. for manifest in {t.manifest for t in Tag.select()}: # Ensure that the manifest's blobs all exist. found_blobs = { b.blob.content_checksum for b in ManifestBlob.select().where( ManifestBlob.manifest == manifest) } parsed = parse_manifest_from_bytes( Bytes.for_string_or_unicode(manifest.manifest_bytes), manifest.media_type.name) assert set(parsed.local_blob_digests) == found_blobs
def pull( self, session, namespace, repo_name, tag_names, images, credentials=None, expected_failure=None, options=None, ): options = options or ProtocolOptions() scopes = options.scopes or ["repository:%s:pull" % self.repo_name(namespace, repo_name)] tag_names = [tag_names] if isinstance(tag_names, str) else tag_names # Ping! self.ping(session) # Perform auth and retrieve a token. token, _ = self.auth( session, credentials, namespace, repo_name, scopes=scopes, expected_failure=expected_failure, ) if token is None and not options.attempt_pull_without_token: return None headers = {} if token: headers = { "Authorization": "Bearer " + token, } if self.schema2: headers["Accept"] = ",".join( options.accept_mimetypes if options.accept_mimetypes is not None else DOCKER_SCHEMA2_CONTENT_TYPES ) manifests = {} image_ids = {} for tag_name in tag_names: # Retrieve the manifest for the tag or digest. response = self.conduct( session, "GET", "/v2/%s/manifests/%s" % (self.repo_name(namespace, repo_name), tag_name), expected_status=(200, expected_failure, V2ProtocolSteps.GET_MANIFEST), headers=headers, ) if response.status_code == 401: assert "WWW-Authenticate" in response.headers response.encoding = "utf-8" if expected_failure is not None: return None # Ensure the manifest returned by us is valid. ct = response.headers["Content-Type"] if not self.schema2: assert ct in DOCKER_SCHEMA1_CONTENT_TYPES manifest = parse_manifest_from_bytes(Bytes.for_string_or_unicode(response.text), ct) manifests[tag_name] = manifest if manifest.schema_version == 1: image_ids[tag_name] = manifest.leaf_layer_v1_image_id # Verify the blobs. layer_index = 0 empty_count = 0 blob_digests = list(manifest.blob_digests) for image in images: if manifest.schema_version == 2 and image.is_empty: empty_count += 1 continue # If the layer is remote, then we expect the blob to *not* exist in the system. blob_digest = blob_digests[layer_index] expected_status = 404 if image.urls else 200 result = self.conduct( session, "GET", "/v2/%s/blobs/%s" % (self.repo_name(namespace, repo_name), blob_digest), expected_status=(expected_status, expected_failure, V2ProtocolSteps.GET_BLOB), headers=headers, options=options, ) if expected_status == 200: assert result.content == image.bytes layer_index += 1 assert (len(blob_digests) + empty_count) >= len( images ) # Schema 2 has 1 extra for config return PullResult(manifests=manifests, image_ids=image_ids)
def pull_list( self, session, namespace, repo_name, tag_names, manifestlist, credentials=None, expected_failure=None, options=None, ): options = options or ProtocolOptions() scopes = options.scopes or [ "repository:%s:push,pull" % self.repo_name(namespace, repo_name) ] tag_names = [tag_names] if isinstance(tag_names, str) else tag_names # Ping! self.ping(session) # Perform auth and retrieve a token. token, _ = self.auth( session, credentials, namespace, repo_name, scopes=scopes, expected_failure=expected_failure, ) if token is None: assert V2Protocol.FAILURE_CODES[V2ProtocolSteps.AUTH].get(expected_failure) return headers = { "Authorization": "Bearer " + token, "Accept": ",".join(DOCKER_SCHEMA2_CONTENT_TYPES), } for tag_name in tag_names: # Retrieve the manifest for the tag or digest. response = self.conduct( session, "GET", "/v2/%s/manifests/%s" % (self.repo_name(namespace, repo_name), tag_name), expected_status=(200, expected_failure, V2ProtocolSteps.GET_MANIFEST_LIST), headers=headers, ) if expected_failure is not None: return None # Parse the returned manifest list and ensure it matches. ct = response.headers["Content-Type"] assert ct == DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE retrieved = parse_manifest_from_bytes(Bytes.for_string_or_unicode(response.text), ct) assert retrieved.schema_version == 2 assert retrieved.is_manifest_list assert retrieved.digest == manifestlist.digest # Pull each of the manifests inside and ensure they can be retrieved. for manifest_digest in retrieved.child_manifest_digests(): response = self.conduct( session, "GET", "/v2/%s/manifests/%s" % (self.repo_name(namespace, repo_name), manifest_digest), expected_status=(200, expected_failure, V2ProtocolSteps.GET_MANIFEST), headers=headers, ) if expected_failure is not None: return None ct = response.headers["Content-Type"] manifest = parse_manifest_from_bytes(Bytes.for_string_or_unicode(response.text), ct) assert not manifest.is_manifest_list assert manifest.digest == manifest_digest
def test_parse_manifest_from_bytes(media_type, manifest_bytes): assert parse_manifest_from_bytes( Bytes.for_string_or_unicode(manifest_bytes), media_type, validate=False)