def _parse_manifest(): content_type = request.content_type or DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE if content_type == "application/json": # For back-compat. content_type = DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE try: return parse_manifest_from_bytes( Bytes.for_string_or_unicode(request.data), content_type) except ManifestException as me: logger.exception("failed to parse manifest when writing by tagname") raise ManifestInvalid( detail={"message": "failed to parse manifest: %s" % me})
def test_renew_manifest_and_parent_tag_when_manifest_is_child_of_manifest_list( self, create_repo, proxy_manifest_response): repo_ref = create_repo(self.orgname, self.upstream_repository, self.user) input_list = parse_manifest_from_bytes( Bytes.for_string_or_unicode(UBI8_LATEST_MANIFEST_LIST), DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE, sparse_manifest_support=True, ) with patch("data.registry_model.registry_proxy_model.Proxy", MagicMock()): proxy_model = ProxyModel( self.orgname, self.upstream_repository, self.user, ) manifest_list, tag = proxy_model._create_manifest_and_retarget_tag( repo_ref, input_list, "latest") assert manifest_list is not None child = (ManifestChild.select(ManifestChild.child_manifest_id).join( Manifest, on=(ManifestChild.child_manifest_id == Manifest.id )).where((ManifestChild.manifest_id == manifest_list.id) & (Manifest.digest == UBI8_LATEST_DIGEST))) manifest_tag = Tag.select().where(Tag.manifest == child).get() manifest_list_tag = tag proxy_mock = proxy_manifest_response( UBI8_LATEST_DIGEST, UBI8_LATEST_MANIFEST_SCHEMA2, DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE) with patch("data.registry_model.registry_proxy_model.Proxy", MagicMock(return_value=proxy_mock)): proxy_model = ProxyModel( self.orgname, self.upstream_repository, self.user, ) manifest = proxy_model.lookup_manifest_by_digest( repo_ref, UBI8_LATEST_DIGEST) updated_tag = oci.tag.get_tag_by_manifest_id(repo_ref.id, manifest.id) updated_list_tag = oci.tag.get_tag_by_manifest_id( repo_ref.id, manifest_list.id) assert updated_tag.id == manifest_tag.id assert updated_list_tag.id == manifest_list_tag.id assert updated_tag.lifetime_end_ms > manifest_tag.lifetime_end_ms assert updated_list_tag.lifetime_end_ms > manifest_list_tag.lifetime_end_ms
def test_does_not_pull_from_upstream_when_manifest_already_exists( self, test_name, proxy_manifest_response): test_params = storage_test_cases[test_name] repo = f"{self.orgname}/{test_params['image_name']}" params = { "repository": repo, "manifest_ref": test_params["manifest_ref"], } r = model.repository.create_repository(self.orgname, test_params["image_name"], self.user) assert r is not None manifest = parse_manifest_from_bytes( Bytes.for_string_or_unicode(test_params["manifest_json"]), test_params["manifest_type"], sparse_manifest_support=True, ) m = oci.manifest.create_manifest(r.id, manifest) assert m is not None if test_params["ref_type"] == "digest": oci.tag.create_temporary_tag_if_necessary(m, 300) else: oci.tag.retarget_tag(test_params["manifest_ref"], m.id) proxy_mock = proxy_manifest_response( test_params["manifest_ref"], test_params["manifest_json"], test_params["manifest_type"], ) with patch("data.registry_model.registry_proxy_model.Proxy", MagicMock(return_value=proxy_mock)): headers = _get_auth_headers(self.sub, self.ctx, repo) headers["Accept"] = ", ".join( DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union( DOCKER_SCHEMA1_CONTENT_TYPES)) conduct_call( self.client, test_params["view_name"], url_for, "GET", params, expected_code=200, headers=headers, ) assert proxy_mock.manifest_exists.call_count == 1 assert proxy_mock.get_manifest.call_count == 0
def test_raises_exception_with_manifest_list(self): manifest = parse_manifest_from_bytes( Bytes.for_string_or_unicode(UBI8_LATEST_MANIFEST_LIST), DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE, ) proxy_model = ProxyModel( self.orgname, self.upstream_repository, self.user, ) with pytest.raises(ManifestException): proxy_model.get_schema1_parsed_manifest( manifest, self.orgname, self.upstream_repository, self.tag, storage, raise_on_error=True, )
def test_create_manifest_and_temp_tag_when_they_dont_exist( self, create_repo): repo_ref = create_repo(self.orgname, self.upstream_repository, self.user) input_manifest = parse_manifest_from_bytes( Bytes.for_string_or_unicode(UBI8_8_4_MANIFEST_SCHEMA2), DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE, ) proxy_model = ProxyModel( self.orgname, self.upstream_repository, self.user, ) manifest, tag = proxy_model._create_manifest_and_retarget_tag( repo_ref, input_manifest, self.tag) assert manifest is not None assert tag is not None assert manifest.internal_manifest_bytes.as_unicode( ) == UBI8_8_4_MANIFEST_SCHEMA2 assert manifest.digest == UBI8_8_4_DIGEST
def test_get_or_create_manifest_invalid_image(initialized_db): repository = get_repository("devtable", "simple") latest_tag = get_tag(repository, "latest") manifest_bytes = Bytes.for_string_or_unicode( latest_tag.manifest.manifest_bytes) parsed = parse_manifest_from_bytes(manifest_bytes, latest_tag.manifest.media_type.name, validate=False) builder = DockerSchema1ManifestBuilder("devtable", "simple", "anothertag") builder.add_layer(parsed.blob_digests[0], '{"id": "foo", "parent": "someinvalidimageid"}') sample_manifest_instance = builder.build(docker_v2_signing_key) created_manifest = get_or_create_manifest(repository, sample_manifest_instance, storage) assert created_manifest is None
def _pull_upstream_manifest(self, repo: str, manifest_ref: str) -> ManifestInterface: try: raw_manifest, content_type = self._proxy.get_manifest( manifest_ref, ACCEPTED_MEDIA_TYPES) except UpstreamRegistryError as e: raise ManifestDoesNotExist(str(e)) upstream_repo_name = self._upstream_repo(repo) upstream_namespace = self._upstream_namespace(repo) # TODO: do we need the compatibility check from v2._parse_manifest? mbytes = Bytes.for_string_or_unicode(raw_manifest) manifest = parse_manifest_from_bytes(mbytes, content_type, sparse_manifest_support=True) valid = self._validate_schema1_manifest(upstream_namespace, upstream_repo_name, manifest) if not valid: raise ManifestDoesNotExist("invalid schema 1 manifest") return manifest
def test_create_sub_manifests_for_manifest_list(self, create_repo): repo_ref = create_repo(self.orgname, self.upstream_repository, self.user) input_manifest = parse_manifest_from_bytes( Bytes.for_string_or_unicode(UBI8_LATEST_MANIFEST_LIST), DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE, sparse_manifest_support=True, ) proxy_model = ProxyModel( self.orgname, self.upstream_repository, self.user, ) manifest, _ = proxy_model._create_manifest_and_retarget_tag( repo_ref, input_manifest, self.tag) mchildren = ManifestChild.select().where( ManifestChild.manifest == manifest.id) created_count = mchildren.count() expected_count = len( list(input_manifest.child_manifests(content_retriever=None))) assert expected_count == created_count
def test_create_manifest_config_blob(self, test_name, proxy_manifest_response): test_params = storage_test_cases[test_name] repo = f"{self.orgname}/{test_params['image_name']}" params = { "repository": repo, "manifest_ref": test_params["manifest_ref"], } proxy_mock = proxy_manifest_response( test_params["manifest_ref"], test_params["manifest_json"], test_params["manifest_type"], ) with patch("data.registry_model.registry_proxy_model.Proxy", MagicMock(return_value=proxy_mock)): headers = _get_auth_headers(self.sub, self.ctx, repo) headers["Accept"] = ", ".join( DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union( DOCKER_SCHEMA1_CONTENT_TYPES)) resp = conduct_call( self.client, test_params["view_name"], url_for, "GET", params, expected_code=200, headers=headers, ) manifest = parse_manifest_from_bytes( Bytes.for_string_or_unicode(test_params["manifest_json"]), test_params["manifest_type"], sparse_manifest_support=True, ) if manifest.schema_version == 2 and not manifest.is_manifest_list: q = ImageStorage.filter( ImageStorage.content_checksum == manifest.config.digest) assert q.count() == 1
def test_connect_existing_blobs_to_new_manifest(self, create_repo): repo_ref = create_repo(self.orgname, self.upstream_repository, self.user) input_manifest = parse_manifest_from_bytes( Bytes.for_string_or_unicode(UBI8_8_4_MANIFEST_SCHEMA2), DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE, ) layer = input_manifest.manifest_dict["layers"][0] blob = ImageStorage.create( image_size=layer["size"], uncompressed_size=layer["size"], content_checksum=layer["digest"], ) proxy_model = ProxyModel( self.orgname, self.upstream_repository, self.user, ) proxy_model._create_manifest_and_retarget_tag(repo_ref, input_manifest, self.tag) blob_count = (ImageStorage.select().where( ImageStorage.content_checksum == blob.content_checksum).count()) assert blob_count == 1
def assert_gc_integrity(expect_storage_removed=True): """ Specialized assertion for ensuring that GC cleans up all dangling storages and labels, invokes the callback for images removed and doesn't invoke the callback for images *not* removed. """ # Add a callback for when images are removed. removed_image_storages = [] remove_callback = model.config.register_image_cleanup_callback( removed_image_storages.extend) # Store existing storages. We won't verify these for existence because they # were likely created as test data. existing_digests = set() for storage_row in ImageStorage.select(): if storage_row.cas_path: existing_digests.add(storage_row.content_checksum) for blob_row in ApprBlob.select(): existing_digests.add(blob_row.digest) # Store the number of dangling objects. existing_storage_count = _get_dangling_storage_count() existing_label_count = _get_dangling_label_count() existing_manifest_count = _get_dangling_manifest_count() # Yield to the GC test. with check_transitive_modifications(): try: yield finally: remove_callback() # Ensure the number of dangling storages, manifests and labels has not changed. updated_storage_count = _get_dangling_storage_count() assert updated_storage_count == existing_storage_count updated_label_count = _get_dangling_label_count() assert updated_label_count == existing_label_count, _get_dangling_labels() updated_manifest_count = _get_dangling_manifest_count() assert updated_manifest_count == existing_manifest_count # Ensure that for each call to the image+storage cleanup callback, the image and its # storage is not found *anywhere* in the database. for removed_image_and_storage in removed_image_storages: assert isinstance(removed_image_and_storage, Image) try: # NOTE: SQLite can and will reuse AUTOINCREMENT IDs occasionally, so if we find a row # with the same ID, make sure it does not have the same Docker Image ID. # See: https://www.sqlite.org/autoinc.html found_image = Image.get(id=removed_image_and_storage.id) assert (found_image.docker_image_id != removed_image_and_storage.docker_image_id ), "Found unexpected removed image %s under repo %s" % ( found_image.id, found_image.repository, ) except Image.DoesNotExist: pass # Ensure that image storages are only removed if not shared. shared = Image.select().where( Image.storage == removed_image_and_storage.storage_id).count() if shared == 0: shared = (ManifestBlob.select().where( ManifestBlob.blob == removed_image_and_storage.storage_id).count()) if shared == 0: shared = (UploadedBlob.select().where( UploadedBlob.blob == removed_image_and_storage.storage_id).count()) if shared == 0: with pytest.raises(ImageStorage.DoesNotExist): ImageStorage.get(id=removed_image_and_storage.storage_id) with pytest.raises(ImageStorage.DoesNotExist): ImageStorage.get(uuid=removed_image_and_storage.storage.uuid) # Ensure all CAS storage is in the storage engine. preferred = storage.preferred_locations[0] for storage_row in ImageStorage.select(): if storage_row.content_checksum in existing_digests: continue if storage_row.cas_path: storage.get_content({preferred}, storage.blob_path( storage_row.content_checksum)) for blob_row in ApprBlob.select(): if blob_row.digest in existing_digests: continue storage.get_content({preferred}, storage.blob_path(blob_row.digest)) # Ensure all tags have valid manifests. for manifest in {t.manifest for t in Tag.select()}: # Ensure that the manifest's blobs all exist. found_blobs = { b.blob.content_checksum for b in ManifestBlob.select().where( ManifestBlob.manifest == manifest) } parsed = parse_manifest_from_bytes( Bytes.for_string_or_unicode(manifest.manifest_bytes), manifest.media_type.name) assert set(parsed.local_blob_digests) == found_blobs
def pull_list( self, session, namespace, repo_name, tag_names, manifestlist, credentials=None, expected_failure=None, options=None, ): options = options or ProtocolOptions() scopes = options.scopes or [ "repository:%s:push,pull" % self.repo_name(namespace, repo_name) ] tag_names = [tag_names] if isinstance(tag_names, str) else tag_names # Ping! self.ping(session) # Perform auth and retrieve a token. token, _ = self.auth( session, credentials, namespace, repo_name, scopes=scopes, expected_failure=expected_failure, ) if token is None: assert V2Protocol.FAILURE_CODES[V2ProtocolSteps.AUTH].get(expected_failure) return headers = { "Authorization": "Bearer " + token, "Accept": ",".join(MANIFEST_LIST_TYPES), } for tag_name in tag_names: # Retrieve the manifest for the tag or digest. response = self.conduct( session, "GET", "/v2/%s/manifests/%s" % (self.repo_name(namespace, repo_name), tag_name), expected_status=(200, expected_failure, V2ProtocolSteps.GET_MANIFEST_LIST), headers=headers, ) if expected_failure is not None: return None # Parse the returned manifest list and ensure it matches. ct = response.headers["Content-Type"] assert is_manifest_list_type(ct), "Expected list type, found: %s" % ct retrieved = parse_manifest_from_bytes(Bytes.for_string_or_unicode(response.text), ct) assert retrieved.schema_version == 2 assert retrieved.is_manifest_list assert retrieved.digest == manifestlist.digest # Pull each of the manifests inside and ensure they can be retrieved. for manifest_digest in retrieved.child_manifest_digests(): response = self.conduct( session, "GET", "/v2/%s/manifests/%s" % (self.repo_name(namespace, repo_name), manifest_digest), expected_status=(200, expected_failure, V2ProtocolSteps.GET_MANIFEST), headers=headers, ) if expected_failure is not None: return None ct = response.headers["Content-Type"] manifest = parse_manifest_from_bytes(Bytes.for_string_or_unicode(response.text), ct) assert not manifest.is_manifest_list assert manifest.digest == manifest_digest
def pull( self, session, namespace, repo_name, tag_names, images, credentials=None, expected_failure=None, options=None, ): options = options or ProtocolOptions() scopes = options.scopes or ["repository:%s:pull" % self.repo_name(namespace, repo_name)] tag_names = [tag_names] if isinstance(tag_names, str) else tag_names # Ping! self.ping(session) # Perform auth and retrieve a token. token, _ = self.auth( session, credentials, namespace, repo_name, scopes=scopes, expected_failure=expected_failure, ) if token is None and not options.attempt_pull_without_token: return None headers = {} if token: headers = { "Authorization": "Bearer " + token, } if self.schema == "oci": headers["Accept"] = ",".join( options.accept_mimetypes if options.accept_mimetypes is not None else OCI_CONTENT_TYPES ) elif self.schema == "schema2": headers["Accept"] = ",".join( options.accept_mimetypes if options.accept_mimetypes is not None else DOCKER_SCHEMA2_CONTENT_TYPES ) manifests = {} image_ids = {} for tag_name in tag_names: # Retrieve the manifest for the tag or digest. response = self.conduct( session, "GET", "/v2/%s/manifests/%s" % (self.repo_name(namespace, repo_name), tag_name), expected_status=(200, expected_failure, V2ProtocolSteps.GET_MANIFEST), headers=headers, ) if response.status_code == 401: assert "WWW-Authenticate" in response.headers response.encoding = "utf-8" if expected_failure is not None: return None # Ensure the manifest returned by us is valid. ct = response.headers["Content-Type"] if self.schema == "schema1": assert ct in DOCKER_SCHEMA1_CONTENT_TYPES if options.require_matching_manifest_type: if self.schema == "schema1": assert ct in DOCKER_SCHEMA1_CONTENT_TYPES if self.schema == "schema2": assert ct in DOCKER_SCHEMA2_CONTENT_TYPES if self.schema == "oci": assert ct in OCI_CONTENT_TYPES manifest = parse_manifest_from_bytes(Bytes.for_string_or_unicode(response.text), ct) manifests[tag_name] = manifest if manifest.schema_version == 1: image_ids[tag_name] = manifest.leaf_layer_v1_image_id # Verify the blobs. layer_index = 0 empty_count = 0 blob_digests = list(manifest.blob_digests) for image in images: if manifest.schema_version == 2 and image.is_empty: empty_count += 1 continue # If the layer is remote, then we expect the blob to *not* exist in the system. blob_digest = blob_digests[layer_index] expected_status = 404 if image.urls else 200 result = self.conduct( session, "GET", "/v2/%s/blobs/%s" % (self.repo_name(namespace, repo_name), blob_digest), expected_status=(expected_status, expected_failure, V2ProtocolSteps.GET_BLOB), headers=headers, options=options, ) if expected_status == 200: assert result.content == image.bytes layer_index += 1 assert (len(blob_digests) + empty_count) >= len( images ) # OCI/Schema 2 has 1 extra for config return PullResult(manifests=manifests, image_ids=image_ids)
def test_manifest_list_create_manifest_with_sub_manifests_and_connect_them( self, test_name, proxy_manifest_response): test_params = storage_test_cases[test_name] if test_params["manifest_type"] not in [ DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE, OCI_IMAGE_INDEX_CONTENT_TYPE, ]: pytest.skip( "regular manifest detected, skipping manifest list specific test." ) repo = f"{self.orgname}/{test_params['image_name']}" params = { "repository": repo, "manifest_ref": test_params["manifest_ref"], } proxy_mock = proxy_manifest_response( test_params["manifest_ref"], test_params["manifest_json"], test_params["manifest_type"], ) with patch("data.registry_model.registry_proxy_model.Proxy", MagicMock(return_value=proxy_mock)): headers = _get_auth_headers(self.sub, self.ctx, repo) headers["Accept"] = ", ".join( DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union( DOCKER_SCHEMA1_CONTENT_TYPES)) conduct_call( self.client, test_params["view_name"], url_for, "GET", params, expected_code=200, headers=headers, ) manifest_list = parse_manifest_from_bytes( Bytes.for_string_or_unicode(test_params["manifest_json"]), test_params["manifest_type"], sparse_manifest_support=True, ) try: manifest = Manifest.filter( Manifest.digest == manifest_list.digest).get() except Manifest.DoesNotExist: assert False, "failed to create manifest list" input_digests = [ manifest["digest"] for manifest in manifest_list.manifest_dict["manifests"] ] manifest_links = ManifestChild.select( ManifestChild.child_manifest).where( ManifestChild.manifest == manifest) sub_digests = [ml.child_manifest.digest for ml in manifest_links] assert input_digests == sub_digests for link in manifest_links: mbytes = link.child_manifest.manifest_bytes assert mbytes == "", f"child manifest bytes expected empty, but got {mbytes}"
def test_pull_placeholder_manifest_updates_manifest_bytes( self, test_name, proxy_manifest_response): """ it's not possible to connect a sub-manifest to a manifest list on a subsequent pull, since a regular manifest request has no pointer to the manifest list it belongs to. to connect the sub-manifests with the manifest list being at pull time, we create a placeholder manifest. placeholder manifests are caracterized by having an empty manifest_bytes. """ test_params = storage_test_cases[test_name] if test_params["manifest_type"] in [ DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE, OCI_IMAGE_INDEX_CONTENT_TYPE, ]: pytest.skip( "manifest list detected, skipping 'flat' manifest specific test." ) # we only have the manifest list json for the hello-world # (because it's significantly smaller). if test_params["image_name"] == "busybox": pytest.skip( "skipping test for busybox image - we do not have its manifest list json." ) if test_params["manifest_type"] == OCI_IMAGE_MANIFEST_CONTENT_TYPE: pytest.skip( "skipping OCI content type - manifest list specifies docker schema v2." ) if test_params["ref_type"] == "tag": pytest.skip( "skipping manifest fetch by tag - pull for a specific architecture is made by digest", ) parsed = parse_manifest_from_bytes( Bytes.for_string_or_unicode(HELLO_WORLD_MANIFEST_LIST_JSON), DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE, sparse_manifest_support=True, ) # first create the manifest list and its placeholders repo = f"{self.orgname}/{test_params['image_name']}" params = { "repository": repo, "manifest_ref": parsed.digest, } proxy_mock = proxy_manifest_response( parsed.digest, HELLO_WORLD_MANIFEST_LIST_JSON, DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE, ) with patch("data.registry_model.registry_proxy_model.Proxy", MagicMock(return_value=proxy_mock)): headers = _get_auth_headers(self.sub, self.ctx, repo) headers["Accept"] = ", ".join( DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union( DOCKER_SCHEMA1_CONTENT_TYPES)) conduct_call( self.client, test_params["view_name"], url_for, "GET", params, expected_code=200, headers=headers, ) # now fetch one of the sub manifests from the manifest list test_manifest = parse_manifest_from_bytes( Bytes.for_string_or_unicode(test_params["manifest_json"]), test_params["manifest_type"], sparse_manifest_support=True, ) params = { "repository": repo, "manifest_ref": test_params["manifest_ref"], } proxy_mock = proxy_manifest_response( test_params["manifest_ref"], test_params["manifest_json"], test_params["manifest_type"], ) with patch("data.registry_model.registry_proxy_model.Proxy", MagicMock(return_value=proxy_mock)): headers = _get_auth_headers(self.sub, self.ctx, repo) headers["Accept"] = ", ".join( DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union( DOCKER_SCHEMA1_CONTENT_TYPES)) resp = conduct_call( self.client, test_params["view_name"], url_for, "GET", params, expected_code=200, headers=headers, ) sub_manifest = Manifest.filter( Manifest.digest == test_params["manifest_ref"]).get() assert sub_manifest.manifest_bytes != "" output_manifest = parse_manifest_from_bytes( Bytes.for_string_or_unicode(sub_manifest.manifest_bytes), sub_manifest.media_type.name, sparse_manifest_support=True, ) input_manifest = parse_manifest_from_bytes( Bytes.for_string_or_unicode(test_params["manifest_json"]), test_params["manifest_type"], sparse_manifest_support=True, ) assert output_manifest.schema_version == input_manifest.schema_version assert output_manifest.media_type == input_manifest.media_type assert output_manifest.is_manifest_list == input_manifest.is_manifest_list assert output_manifest.digest == input_manifest.digest assert output_manifest.manifest_dict == input_manifest.manifest_dict
def test_parse_manifest_from_bytes(media_type, manifest_bytes): assert parse_manifest_from_bytes( Bytes.for_string_or_unicode(manifest_bytes), media_type, validate=False)
def test_create_placeholder_blobs_on_first_pull(self, test_name, proxy_manifest_response): test_params = storage_test_cases[test_name] # no blob placeholders are created for manifest lists - we don't have # the sub-manifests at manifest list creation time, so there's no way # to know which blobs the sub-manifest has. if test_params["manifest_type"] in [ DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE, OCI_IMAGE_INDEX_CONTENT_TYPE, ]: pytest.skip( "manifest list detected - skipping blob placeholder test") repo = f"{self.orgname}/{test_params['image_name']}" params = { "repository": repo, "manifest_ref": test_params["manifest_ref"], } proxy_mock = proxy_manifest_response(test_params["manifest_ref"], test_params["manifest_json"], test_params["manifest_type"]) with patch("data.registry_model.registry_proxy_model.Proxy", MagicMock(return_value=proxy_mock)): headers = _get_auth_headers(self.sub, self.ctx, repo) headers["Accept"] = ", ".join( DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union( DOCKER_SCHEMA1_CONTENT_TYPES)) conduct_call( self.client, test_params["view_name"], url_for, "GET", params, expected_code=200, headers=headers, ) parsed = parse_manifest_from_bytes( Bytes.for_string_or_unicode(test_params["manifest_json"]), test_params["manifest_type"], sparse_manifest_support=True, ) manifest = Manifest.filter(Manifest.digest == parsed.digest).get() mdict = parsed.manifest_dict layers = mdict.get("layers", mdict.get("fsLayers")) mblobs = ManifestBlob.filter(ManifestBlob.manifest == manifest) expected_count = len(layers) # schema 2 manifests have an extra config blob which we need to take into # consideration in the total count config_digest = "" if parsed.schema_version == 2: config_digest = parsed.config.digest expected_count += 1 assert mblobs.count() == expected_count for mblob in mblobs: blob = None layer = None # don't assert if digest belongs to a config blob if mblob.blob.content_checksum == config_digest: continue for layer in layers: digest = layer.get("digest", layer.get("blobSum")) if mblob.blob.content_checksum == digest: blob = mblob.blob layer = layer break assert blob is not None assert blob.image_size == layer.get("size", None) # the absence of an image storage placement for a blob indicates that it's # a placeholder blob, not yet downloaded from the upstream registry. placements = ImageStoragePlacement.filter( ImageStoragePlacement.storage == blob) assert placements.count() == 0
def assert_gc_integrity(expect_storage_removed=True): """ Specialized assertion for ensuring that GC cleans up all dangling storages and labels, invokes the callback for images removed and doesn't invoke the callback for images *not* removed. """ # Add a callback for when images are removed. removed_image_storages = [] model.config.register_image_cleanup_callback(removed_image_storages.extend) # Store existing storages. We won't verify these for existence because they # were likely created as test data. existing_digests = set() for storage_row in ImageStorage.select(): if storage_row.cas_path: existing_digests.add(storage_row.content_checksum) for blob_row in ApprBlob.select(): existing_digests.add(blob_row.digest) # Store the number of dangling storages and labels. existing_storage_count = _get_dangling_storage_count() existing_label_count = _get_dangling_label_count() existing_manifest_count = _get_dangling_manifest_count() # Yield to the GC test. with check_transitive_modifications(): yield # Ensure the number of dangling storages, manifests and labels has not changed. updated_storage_count = _get_dangling_storage_count() assert updated_storage_count == existing_storage_count updated_label_count = _get_dangling_label_count() assert updated_label_count == existing_label_count, _get_dangling_labels() updated_manifest_count = _get_dangling_manifest_count() assert updated_manifest_count == existing_manifest_count # Ensure that for each call to the image+storage cleanup callback, the image and its # storage is not found *anywhere* in the database. for removed_image_and_storage in removed_image_storages: with pytest.raises(Image.DoesNotExist): Image.get(id=removed_image_and_storage.id) # Ensure that image storages are only removed if not shared. shared = Image.select().where( Image.storage == removed_image_and_storage.storage_id).count() if shared == 0: shared = (ManifestBlob.select().where( ManifestBlob.blob == removed_image_and_storage.storage_id).count()) if shared == 0: with pytest.raises(ImageStorage.DoesNotExist): ImageStorage.get(id=removed_image_and_storage.storage_id) with pytest.raises(ImageStorage.DoesNotExist): ImageStorage.get(uuid=removed_image_and_storage.storage.uuid) # Ensure all CAS storage is in the storage engine. preferred = storage.preferred_locations[0] for storage_row in ImageStorage.select(): if storage_row.content_checksum in existing_digests: continue if storage_row.cas_path: storage.get_content({preferred}, storage.blob_path( storage_row.content_checksum)) for blob_row in ApprBlob.select(): if blob_row.digest in existing_digests: continue storage.get_content({preferred}, storage.blob_path(blob_row.digest)) # Ensure all tags have valid manifests. for manifest in {t.manifest for t in Tag.select()}: # Ensure that the manifest's blobs all exist. found_blobs = { b.blob.content_checksum for b in ManifestBlob.select().where( ManifestBlob.manifest == manifest) } parsed = parse_manifest_from_bytes( Bytes.for_string_or_unicode(manifest.manifest_bytes), manifest.media_type.name) assert set(parsed.local_blob_digests) == found_blobs