def test_does_not_create_repo_when_upstream_repo_does_not_exist( self, test_name, proxy_manifest_response): test_params = storage_test_cases[test_name] repo = f"{self.orgname}/{test_params['image_name']}" params = { "repository": repo, "manifest_ref": test_params["manifest_ref"], } proxy_mock = proxy_manifest_response("not-existing-ref", "", "") with patch("data.registry_model.registry_proxy_model.Proxy", MagicMock(return_value=proxy_mock)): headers = _get_auth_headers(self.sub, self.ctx, repo) headers["Accept"] = ", ".join( DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union( DOCKER_SCHEMA1_CONTENT_TYPES)) conduct_call( self.client, test_params["view_name"], url_for, "GET", params, expected_code=404, headers=headers, ) count = Repository.filter( Repository.name == test_params["image_name"], Repository.namespace_user == self.org.id).count() assert count == 0
def test_creates_repo_on_first_pull(self, test_name, proxy_manifest_response): test_params = storage_test_cases[test_name] repo = f"{self.orgname}/{test_params['image_name']}" params = { "repository": repo, "manifest_ref": test_params["manifest_ref"], } proxy_mock = proxy_manifest_response( test_params["manifest_ref"], test_params["manifest_json"], test_params["manifest_type"], ) with patch("data.registry_model.registry_proxy_model.Proxy", MagicMock(return_value=proxy_mock)): headers = _get_auth_headers(self.sub, self.ctx, repo) headers["Accept"] = ", ".join( DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union( DOCKER_SCHEMA1_CONTENT_TYPES)) conduct_call( self.client, test_params["view_name"], url_for, "GET", params, expected_code=200, headers=headers, ) repo = model.repository.get_repository(self.orgname, test_params["image_name"]) assert repo is not None assert repo.visibility.name == "private"
def test_creates_manifest_on_first_pull(self, test_name, proxy_manifest_response): test_params = storage_test_cases[test_name] repo = f"{self.orgname}/{test_params['image_name']}" params = { "repository": repo, "manifest_ref": test_params["manifest_ref"], } proxy_mock = proxy_manifest_response( test_params["manifest_ref"], test_params["manifest_json"], test_params["manifest_type"], ) with patch("data.registry_model.registry_proxy_model.Proxy", MagicMock(return_value=proxy_mock)): headers = _get_auth_headers(self.sub, self.ctx, repo) headers["Accept"] = ", ".join( DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union( DOCKER_SCHEMA1_CONTENT_TYPES)) conduct_call( self.client, test_params["view_name"], url_for, "GET", params, expected_code=200, headers=headers, ) repository_ref = registry_model.lookup_repository( self.orgname, test_params["image_name"]) assert repository_ref is not None tag = registry_model.get_repo_tag(repository_ref, test_params["manifest_ref"]) # when testing the fetch_manifest_by_digest view the tag created is temporary, # and it does not refer to the manifest digest (manifest_ref), so we need to # fetch it by its link to the repository instead. if test_params["ref_type"] == "digest": tag = Tag.filter(Tag.repository_id == repository_ref.id).get() # get_manifest_for_tag returns a tag of datatypes.Tag, so we convert # the one we have to that type. tag = datatypes.Tag.for_tag(tag, SyntheticIDHandler()) assert tag is not None manifest = registry_model.get_manifest_for_tag(tag) assert manifest is not None output_manifest = manifest.get_parsed_manifest() input_manifest = parse_manifest_from_bytes( Bytes.for_string_or_unicode(test_params["manifest_json"]), test_params["manifest_type"], sparse_manifest_support=True, ) assert output_manifest.schema_version == input_manifest.schema_version assert output_manifest.media_type == input_manifest.media_type assert output_manifest.is_manifest_list == input_manifest.is_manifest_list assert output_manifest.digest == input_manifest.digest assert output_manifest.manifest_dict == input_manifest.manifest_dict
def test_does_not_pull_from_upstream_when_manifest_already_exists( self, test_name, proxy_manifest_response): test_params = storage_test_cases[test_name] repo = f"{self.orgname}/{test_params['image_name']}" params = { "repository": repo, "manifest_ref": test_params["manifest_ref"], } r = model.repository.create_repository(self.orgname, test_params["image_name"], self.user) assert r is not None manifest = parse_manifest_from_bytes( Bytes.for_string_or_unicode(test_params["manifest_json"]), test_params["manifest_type"], sparse_manifest_support=True, ) m = oci.manifest.create_manifest(r.id, manifest) assert m is not None if test_params["ref_type"] == "digest": oci.tag.create_temporary_tag_if_necessary(m, 300) else: oci.tag.retarget_tag(test_params["manifest_ref"], m.id) proxy_mock = proxy_manifest_response( test_params["manifest_ref"], test_params["manifest_json"], test_params["manifest_type"], ) with patch("data.registry_model.registry_proxy_model.Proxy", MagicMock(return_value=proxy_mock)): headers = _get_auth_headers(self.sub, self.ctx, repo) headers["Accept"] = ", ".join( DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union( DOCKER_SCHEMA1_CONTENT_TYPES)) conduct_call( self.client, test_params["view_name"], url_for, "GET", params, expected_code=200, headers=headers, ) assert proxy_mock.manifest_exists.call_count == 1 assert proxy_mock.get_manifest.call_count == 0
def test_pull_proxy_whole_dockerhub_404(self): params = { "repository": self.repository, "manifest_ref": "666", } headers = self._get_auth_headers(self.repository) headers["Accept"] = ", ".join( DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union( DOCKER_SCHEMA1_CONTENT_TYPES)) conduct_call( self.client, "v2.fetch_manifest_by_tagname", url_for, "GET", params, expected_code=404, headers=headers, )
def test_check_manifest_exists_by_tag_404(self): params = { "repository": self.repository, "manifest_ref": "666", } headers = _get_auth_headers(self.sub, self.ctx, self.repository) headers["Accept"] = ", ".join( DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union( DOCKER_SCHEMA1_CONTENT_TYPES)) conduct_call( self.client, "v2.fetch_manifest_by_tagname", url_for, "HEAD", params, expected_code=404, headers=headers, )
def test_pull_proxy_single_namespace(self): params = { "repository": self.repository2, "manifest_ref": self.tag, } headers = _get_auth_headers(self.sub, self.ctx, self.repository2) headers["Accept"] = ", ".join( DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union( DOCKER_SCHEMA1_CONTENT_TYPES)) conduct_call( self.client, "v2.fetch_manifest_by_tagname", url_for, "GET", params, expected_code=200, headers=headers, )
def test_check_manifest_exists_from_dockerhub_by_tag(self): params = { "repository": self.repository, "manifest_ref": self.tag, } headers = self._get_auth_headers(self.repository) headers["Accept"] = ", ".join( DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union( DOCKER_SCHEMA1_CONTENT_TYPES)) conduct_call( self.client, "v2.fetch_manifest_by_tagname", url_for, "HEAD", params, expected_code=200, headers=headers, )
def test_create_manifest_config_blob(self, test_name, proxy_manifest_response): test_params = storage_test_cases[test_name] repo = f"{self.orgname}/{test_params['image_name']}" params = { "repository": repo, "manifest_ref": test_params["manifest_ref"], } proxy_mock = proxy_manifest_response( test_params["manifest_ref"], test_params["manifest_json"], test_params["manifest_type"], ) with patch("data.registry_model.registry_proxy_model.Proxy", MagicMock(return_value=proxy_mock)): headers = _get_auth_headers(self.sub, self.ctx, repo) headers["Accept"] = ", ".join( DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union( DOCKER_SCHEMA1_CONTENT_TYPES)) resp = conduct_call( self.client, test_params["view_name"], url_for, "GET", params, expected_code=200, headers=headers, ) manifest = parse_manifest_from_bytes( Bytes.for_string_or_unicode(test_params["manifest_json"]), test_params["manifest_type"], sparse_manifest_support=True, ) if manifest.schema_version == 2 and not manifest.is_manifest_list: q = ImageStorage.filter( ImageStorage.content_checksum == manifest.config.digest) assert q.count() == 1
def test_create_placeholder_blobs_on_first_pull(self, test_name, proxy_manifest_response): test_params = storage_test_cases[test_name] # no blob placeholders are created for manifest lists - we don't have # the sub-manifests at manifest list creation time, so there's no way # to know which blobs the sub-manifest has. if test_params["manifest_type"] in [ DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE, OCI_IMAGE_INDEX_CONTENT_TYPE, ]: pytest.skip( "manifest list detected - skipping blob placeholder test") repo = f"{self.orgname}/{test_params['image_name']}" params = { "repository": repo, "manifest_ref": test_params["manifest_ref"], } proxy_mock = proxy_manifest_response(test_params["manifest_ref"], test_params["manifest_json"], test_params["manifest_type"]) with patch("data.registry_model.registry_proxy_model.Proxy", MagicMock(return_value=proxy_mock)): headers = _get_auth_headers(self.sub, self.ctx, repo) headers["Accept"] = ", ".join( DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union( DOCKER_SCHEMA1_CONTENT_TYPES)) conduct_call( self.client, test_params["view_name"], url_for, "GET", params, expected_code=200, headers=headers, ) parsed = parse_manifest_from_bytes( Bytes.for_string_or_unicode(test_params["manifest_json"]), test_params["manifest_type"], sparse_manifest_support=True, ) manifest = Manifest.filter(Manifest.digest == parsed.digest).get() mdict = parsed.manifest_dict layers = mdict.get("layers", mdict.get("fsLayers")) mblobs = ManifestBlob.filter(ManifestBlob.manifest == manifest) expected_count = len(layers) # schema 2 manifests have an extra config blob which we need to take into # consideration in the total count config_digest = "" if parsed.schema_version == 2: config_digest = parsed.config.digest expected_count += 1 assert mblobs.count() == expected_count for mblob in mblobs: blob = None layer = None # don't assert if digest belongs to a config blob if mblob.blob.content_checksum == config_digest: continue for layer in layers: digest = layer.get("digest", layer.get("blobSum")) if mblob.blob.content_checksum == digest: blob = mblob.blob layer = layer break assert blob is not None assert blob.image_size == layer.get("size", None) # the absence of an image storage placement for a blob indicates that it's # a placeholder blob, not yet downloaded from the upstream registry. placements = ImageStoragePlacement.filter( ImageStoragePlacement.storage == blob) assert placements.count() == 0
def test_pull_placeholder_manifest_updates_manifest_bytes( self, test_name, proxy_manifest_response): """ it's not possible to connect a sub-manifest to a manifest list on a subsequent pull, since a regular manifest request has no pointer to the manifest list it belongs to. to connect the sub-manifests with the manifest list being at pull time, we create a placeholder manifest. placeholder manifests are caracterized by having an empty manifest_bytes. """ test_params = storage_test_cases[test_name] if test_params["manifest_type"] in [ DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE, OCI_IMAGE_INDEX_CONTENT_TYPE, ]: pytest.skip( "manifest list detected, skipping 'flat' manifest specific test." ) # we only have the manifest list json for the hello-world # (because it's significantly smaller). if test_params["image_name"] == "busybox": pytest.skip( "skipping test for busybox image - we do not have its manifest list json." ) if test_params["manifest_type"] == OCI_IMAGE_MANIFEST_CONTENT_TYPE: pytest.skip( "skipping OCI content type - manifest list specifies docker schema v2." ) if test_params["ref_type"] == "tag": pytest.skip( "skipping manifest fetch by tag - pull for a specific architecture is made by digest", ) parsed = parse_manifest_from_bytes( Bytes.for_string_or_unicode(HELLO_WORLD_MANIFEST_LIST_JSON), DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE, sparse_manifest_support=True, ) # first create the manifest list and its placeholders repo = f"{self.orgname}/{test_params['image_name']}" params = { "repository": repo, "manifest_ref": parsed.digest, } proxy_mock = proxy_manifest_response( parsed.digest, HELLO_WORLD_MANIFEST_LIST_JSON, DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE, ) with patch("data.registry_model.registry_proxy_model.Proxy", MagicMock(return_value=proxy_mock)): headers = _get_auth_headers(self.sub, self.ctx, repo) headers["Accept"] = ", ".join( DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union( DOCKER_SCHEMA1_CONTENT_TYPES)) conduct_call( self.client, test_params["view_name"], url_for, "GET", params, expected_code=200, headers=headers, ) # now fetch one of the sub manifests from the manifest list test_manifest = parse_manifest_from_bytes( Bytes.for_string_or_unicode(test_params["manifest_json"]), test_params["manifest_type"], sparse_manifest_support=True, ) params = { "repository": repo, "manifest_ref": test_params["manifest_ref"], } proxy_mock = proxy_manifest_response( test_params["manifest_ref"], test_params["manifest_json"], test_params["manifest_type"], ) with patch("data.registry_model.registry_proxy_model.Proxy", MagicMock(return_value=proxy_mock)): headers = _get_auth_headers(self.sub, self.ctx, repo) headers["Accept"] = ", ".join( DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union( DOCKER_SCHEMA1_CONTENT_TYPES)) resp = conduct_call( self.client, test_params["view_name"], url_for, "GET", params, expected_code=200, headers=headers, ) sub_manifest = Manifest.filter( Manifest.digest == test_params["manifest_ref"]).get() assert sub_manifest.manifest_bytes != "" output_manifest = parse_manifest_from_bytes( Bytes.for_string_or_unicode(sub_manifest.manifest_bytes), sub_manifest.media_type.name, sparse_manifest_support=True, ) input_manifest = parse_manifest_from_bytes( Bytes.for_string_or_unicode(test_params["manifest_json"]), test_params["manifest_type"], sparse_manifest_support=True, ) assert output_manifest.schema_version == input_manifest.schema_version assert output_manifest.media_type == input_manifest.media_type assert output_manifest.is_manifest_list == input_manifest.is_manifest_list assert output_manifest.digest == input_manifest.digest assert output_manifest.manifest_dict == input_manifest.manifest_dict
def test_manifest_list_create_manifest_with_sub_manifests_and_connect_them( self, test_name, proxy_manifest_response): test_params = storage_test_cases[test_name] if test_params["manifest_type"] not in [ DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE, OCI_IMAGE_INDEX_CONTENT_TYPE, ]: pytest.skip( "regular manifest detected, skipping manifest list specific test." ) repo = f"{self.orgname}/{test_params['image_name']}" params = { "repository": repo, "manifest_ref": test_params["manifest_ref"], } proxy_mock = proxy_manifest_response( test_params["manifest_ref"], test_params["manifest_json"], test_params["manifest_type"], ) with patch("data.registry_model.registry_proxy_model.Proxy", MagicMock(return_value=proxy_mock)): headers = _get_auth_headers(self.sub, self.ctx, repo) headers["Accept"] = ", ".join( DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union( DOCKER_SCHEMA1_CONTENT_TYPES)) conduct_call( self.client, test_params["view_name"], url_for, "GET", params, expected_code=200, headers=headers, ) manifest_list = parse_manifest_from_bytes( Bytes.for_string_or_unicode(test_params["manifest_json"]), test_params["manifest_type"], sparse_manifest_support=True, ) try: manifest = Manifest.filter( Manifest.digest == manifest_list.digest).get() except Manifest.DoesNotExist: assert False, "failed to create manifest list" input_digests = [ manifest["digest"] for manifest in manifest_list.manifest_dict["manifests"] ] manifest_links = ManifestChild.select( ManifestChild.child_manifest).where( ManifestChild.manifest == manifest) sub_digests = [ml.child_manifest.digest for ml in manifest_links] assert input_digests == sub_digests for link in manifest_links: mbytes = link.child_manifest.manifest_bytes assert mbytes == "", f"child manifest bytes expected empty, but got {mbytes}"