def test_does_not_create_repo_when_upstream_repo_does_not_exist(
self, test_name, proxy_manifest_response):
test_params = storage_test_cases[test_name]
repo = f"{self.orgname}/{test_params['image_name']}"
params = {
"repository": repo,
"manifest_ref": test_params["manifest_ref"],
}
proxy_mock = proxy_manifest_response("not-existing-ref", "", "")
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
headers = _get_auth_headers(self.sub, self.ctx, repo)
headers["Accept"] = ", ".join(
DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union(
DOCKER_SCHEMA1_CONTENT_TYPES))
conduct_call(
self.client,
test_params["view_name"],
url_for,
"GET",
params,
expected_code=404,
headers=headers,
)
count = Repository.filter(
Repository.name == test_params["image_name"],
Repository.namespace_user == self.org.id).count()
assert count == 0
def test_creates_repo_on_first_pull(self, test_name,
proxy_manifest_response):
test_params = storage_test_cases[test_name]
repo = f"{self.orgname}/{test_params['image_name']}"
params = {
"repository": repo,
"manifest_ref": test_params["manifest_ref"],
}
proxy_mock = proxy_manifest_response(
test_params["manifest_ref"],
test_params["manifest_json"],
test_params["manifest_type"],
)
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
headers = _get_auth_headers(self.sub, self.ctx, repo)
headers["Accept"] = ", ".join(
DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union(
DOCKER_SCHEMA1_CONTENT_TYPES))
conduct_call(
self.client,
test_params["view_name"],
url_for,
"GET",
params,
expected_code=200,
headers=headers,
)
repo = model.repository.get_repository(self.orgname,
test_params["image_name"])
assert repo is not None
assert repo.visibility.name == "private"
def test_creates_manifest_on_first_pull(self, test_name,
proxy_manifest_response):
test_params = storage_test_cases[test_name]
repo = f"{self.orgname}/{test_params['image_name']}"
params = {
"repository": repo,
"manifest_ref": test_params["manifest_ref"],
}
proxy_mock = proxy_manifest_response(
test_params["manifest_ref"],
test_params["manifest_json"],
test_params["manifest_type"],
)
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
headers = _get_auth_headers(self.sub, self.ctx, repo)
headers["Accept"] = ", ".join(
DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union(
DOCKER_SCHEMA1_CONTENT_TYPES))
conduct_call(
self.client,
test_params["view_name"],
url_for,
"GET",
params,
expected_code=200,
headers=headers,
)
repository_ref = registry_model.lookup_repository(
self.orgname, test_params["image_name"])
assert repository_ref is not None
tag = registry_model.get_repo_tag(repository_ref,
test_params["manifest_ref"])
# when testing the fetch_manifest_by_digest view the tag created is temporary,
# and it does not refer to the manifest digest (manifest_ref), so we need to
# fetch it by its link to the repository instead.
if test_params["ref_type"] == "digest":
tag = Tag.filter(Tag.repository_id == repository_ref.id).get()
# get_manifest_for_tag returns a tag of datatypes.Tag, so we convert
# the one we have to that type.
tag = datatypes.Tag.for_tag(tag, SyntheticIDHandler())
assert tag is not None
manifest = registry_model.get_manifest_for_tag(tag)
assert manifest is not None
output_manifest = manifest.get_parsed_manifest()
input_manifest = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(test_params["manifest_json"]),
test_params["manifest_type"],
sparse_manifest_support=True,
)
assert output_manifest.schema_version == input_manifest.schema_version
assert output_manifest.media_type == input_manifest.media_type
assert output_manifest.is_manifest_list == input_manifest.is_manifest_list
assert output_manifest.digest == input_manifest.digest
assert output_manifest.manifest_dict == input_manifest.manifest_dict
def test_does_not_pull_from_upstream_when_manifest_already_exists(
self, test_name, proxy_manifest_response):
test_params = storage_test_cases[test_name]
repo = f"{self.orgname}/{test_params['image_name']}"
params = {
"repository": repo,
"manifest_ref": test_params["manifest_ref"],
}
r = model.repository.create_repository(self.orgname,
test_params["image_name"],
self.user)
assert r is not None
manifest = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(test_params["manifest_json"]),
test_params["manifest_type"],
sparse_manifest_support=True,
)
m = oci.manifest.create_manifest(r.id, manifest)
assert m is not None
if test_params["ref_type"] == "digest":
oci.tag.create_temporary_tag_if_necessary(m, 300)
else:
oci.tag.retarget_tag(test_params["manifest_ref"], m.id)
proxy_mock = proxy_manifest_response(
test_params["manifest_ref"],
test_params["manifest_json"],
test_params["manifest_type"],
)
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
headers = _get_auth_headers(self.sub, self.ctx, repo)
headers["Accept"] = ", ".join(
DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union(
DOCKER_SCHEMA1_CONTENT_TYPES))
conduct_call(
self.client,
test_params["view_name"],
url_for,
"GET",
params,
expected_code=200,
headers=headers,
)
assert proxy_mock.manifest_exists.call_count == 1
assert proxy_mock.get_manifest.call_count == 0
def test_pull_proxy_whole_dockerhub_404(self):
params = {
"repository": self.repository,
"manifest_ref": "666",
}
headers = self._get_auth_headers(self.repository)
headers["Accept"] = ", ".join(
DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union(
DOCKER_SCHEMA1_CONTENT_TYPES))
conduct_call(
self.client,
"v2.fetch_manifest_by_tagname",
url_for,
"GET",
params,
expected_code=404,
headers=headers,
)
def test_check_manifest_exists_by_tag_404(self):
params = {
"repository": self.repository,
"manifest_ref": "666",
}
headers = _get_auth_headers(self.sub, self.ctx, self.repository)
headers["Accept"] = ", ".join(
DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union(
DOCKER_SCHEMA1_CONTENT_TYPES))
conduct_call(
self.client,
"v2.fetch_manifest_by_tagname",
url_for,
"HEAD",
params,
expected_code=404,
headers=headers,
)
def test_pull_proxy_single_namespace(self):
params = {
"repository": self.repository2,
"manifest_ref": self.tag,
}
headers = _get_auth_headers(self.sub, self.ctx, self.repository2)
headers["Accept"] = ", ".join(
DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union(
DOCKER_SCHEMA1_CONTENT_TYPES))
conduct_call(
self.client,
"v2.fetch_manifest_by_tagname",
url_for,
"GET",
params,
expected_code=200,
headers=headers,
)
def test_check_manifest_exists_from_dockerhub_by_tag(self):
params = {
"repository": self.repository,
"manifest_ref": self.tag,
}
headers = self._get_auth_headers(self.repository)
headers["Accept"] = ", ".join(
DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union(
DOCKER_SCHEMA1_CONTENT_TYPES))
conduct_call(
self.client,
"v2.fetch_manifest_by_tagname",
url_for,
"HEAD",
params,
expected_code=200,
headers=headers,
)
def test_create_manifest_config_blob(self, test_name,
proxy_manifest_response):
test_params = storage_test_cases[test_name]
repo = f"{self.orgname}/{test_params['image_name']}"
params = {
"repository": repo,
"manifest_ref": test_params["manifest_ref"],
}
proxy_mock = proxy_manifest_response(
test_params["manifest_ref"],
test_params["manifest_json"],
test_params["manifest_type"],
)
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
headers = _get_auth_headers(self.sub, self.ctx, repo)
headers["Accept"] = ", ".join(
DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union(
DOCKER_SCHEMA1_CONTENT_TYPES))
resp = conduct_call(
self.client,
test_params["view_name"],
url_for,
"GET",
params,
expected_code=200,
headers=headers,
)
manifest = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(test_params["manifest_json"]),
test_params["manifest_type"],
sparse_manifest_support=True,
)
if manifest.schema_version == 2 and not manifest.is_manifest_list:
q = ImageStorage.filter(
ImageStorage.content_checksum == manifest.config.digest)
assert q.count() == 1
def test_create_placeholder_blobs_on_first_pull(self, test_name,
proxy_manifest_response):
test_params = storage_test_cases[test_name]
# no blob placeholders are created for manifest lists - we don't have
# the sub-manifests at manifest list creation time, so there's no way
# to know which blobs the sub-manifest has.
if test_params["manifest_type"] in [
DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE,
OCI_IMAGE_INDEX_CONTENT_TYPE,
]:
pytest.skip(
"manifest list detected - skipping blob placeholder test")
repo = f"{self.orgname}/{test_params['image_name']}"
params = {
"repository": repo,
"manifest_ref": test_params["manifest_ref"],
}
proxy_mock = proxy_manifest_response(test_params["manifest_ref"],
test_params["manifest_json"],
test_params["manifest_type"])
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
headers = _get_auth_headers(self.sub, self.ctx, repo)
headers["Accept"] = ", ".join(
DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union(
DOCKER_SCHEMA1_CONTENT_TYPES))
conduct_call(
self.client,
test_params["view_name"],
url_for,
"GET",
params,
expected_code=200,
headers=headers,
)
parsed = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(test_params["manifest_json"]),
test_params["manifest_type"],
sparse_manifest_support=True,
)
manifest = Manifest.filter(Manifest.digest == parsed.digest).get()
mdict = parsed.manifest_dict
layers = mdict.get("layers", mdict.get("fsLayers"))
mblobs = ManifestBlob.filter(ManifestBlob.manifest == manifest)
expected_count = len(layers)
# schema 2 manifests have an extra config blob which we need to take into
# consideration in the total count
config_digest = ""
if parsed.schema_version == 2:
config_digest = parsed.config.digest
expected_count += 1
assert mblobs.count() == expected_count
for mblob in mblobs:
blob = None
layer = None
# don't assert if digest belongs to a config blob
if mblob.blob.content_checksum == config_digest:
continue
for layer in layers:
digest = layer.get("digest", layer.get("blobSum"))
if mblob.blob.content_checksum == digest:
blob = mblob.blob
layer = layer
break
assert blob is not None
assert blob.image_size == layer.get("size", None)
# the absence of an image storage placement for a blob indicates that it's
# a placeholder blob, not yet downloaded from the upstream registry.
placements = ImageStoragePlacement.filter(
ImageStoragePlacement.storage == blob)
assert placements.count() == 0
def test_pull_placeholder_manifest_updates_manifest_bytes(
self, test_name, proxy_manifest_response):
"""
it's not possible to connect a sub-manifest to a manifest list on a subsequent pull,
since a regular manifest request has no pointer to the manifest list it belongs to.
to connect the sub-manifests with the manifest list being at pull time, we create a
placeholder manifest.
placeholder manifests are caracterized by having an empty manifest_bytes.
"""
test_params = storage_test_cases[test_name]
if test_params["manifest_type"] in [
DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE,
OCI_IMAGE_INDEX_CONTENT_TYPE,
]:
pytest.skip(
"manifest list detected, skipping 'flat' manifest specific test."
)
# we only have the manifest list json for the hello-world
# (because it's significantly smaller).
if test_params["image_name"] == "busybox":
pytest.skip(
"skipping test for busybox image - we do not have its manifest list json."
)
if test_params["manifest_type"] == OCI_IMAGE_MANIFEST_CONTENT_TYPE:
pytest.skip(
"skipping OCI content type - manifest list specifies docker schema v2."
)
if test_params["ref_type"] == "tag":
pytest.skip(
"skipping manifest fetch by tag - pull for a specific architecture is made by digest",
)
parsed = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(HELLO_WORLD_MANIFEST_LIST_JSON),
DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE,
sparse_manifest_support=True,
)
# first create the manifest list and its placeholders
repo = f"{self.orgname}/{test_params['image_name']}"
params = {
"repository": repo,
"manifest_ref": parsed.digest,
}
proxy_mock = proxy_manifest_response(
parsed.digest,
HELLO_WORLD_MANIFEST_LIST_JSON,
DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE,
)
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
headers = _get_auth_headers(self.sub, self.ctx, repo)
headers["Accept"] = ", ".join(
DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union(
DOCKER_SCHEMA1_CONTENT_TYPES))
conduct_call(
self.client,
test_params["view_name"],
url_for,
"GET",
params,
expected_code=200,
headers=headers,
)
# now fetch one of the sub manifests from the manifest list
test_manifest = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(test_params["manifest_json"]),
test_params["manifest_type"],
sparse_manifest_support=True,
)
params = {
"repository": repo,
"manifest_ref": test_params["manifest_ref"],
}
proxy_mock = proxy_manifest_response(
test_params["manifest_ref"],
test_params["manifest_json"],
test_params["manifest_type"],
)
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
headers = _get_auth_headers(self.sub, self.ctx, repo)
headers["Accept"] = ", ".join(
DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union(
DOCKER_SCHEMA1_CONTENT_TYPES))
resp = conduct_call(
self.client,
test_params["view_name"],
url_for,
"GET",
params,
expected_code=200,
headers=headers,
)
sub_manifest = Manifest.filter(
Manifest.digest == test_params["manifest_ref"]).get()
assert sub_manifest.manifest_bytes != ""
output_manifest = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(sub_manifest.manifest_bytes),
sub_manifest.media_type.name,
sparse_manifest_support=True,
)
input_manifest = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(test_params["manifest_json"]),
test_params["manifest_type"],
sparse_manifest_support=True,
)
assert output_manifest.schema_version == input_manifest.schema_version
assert output_manifest.media_type == input_manifest.media_type
assert output_manifest.is_manifest_list == input_manifest.is_manifest_list
assert output_manifest.digest == input_manifest.digest
assert output_manifest.manifest_dict == input_manifest.manifest_dict
def test_manifest_list_create_manifest_with_sub_manifests_and_connect_them(
self, test_name, proxy_manifest_response):
test_params = storage_test_cases[test_name]
if test_params["manifest_type"] not in [
DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE,
OCI_IMAGE_INDEX_CONTENT_TYPE,
]:
pytest.skip(
"regular manifest detected, skipping manifest list specific test."
)
repo = f"{self.orgname}/{test_params['image_name']}"
params = {
"repository": repo,
"manifest_ref": test_params["manifest_ref"],
}
proxy_mock = proxy_manifest_response(
test_params["manifest_ref"],
test_params["manifest_json"],
test_params["manifest_type"],
)
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
headers = _get_auth_headers(self.sub, self.ctx, repo)
headers["Accept"] = ", ".join(
DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union(
DOCKER_SCHEMA1_CONTENT_TYPES))
conduct_call(
self.client,
test_params["view_name"],
url_for,
"GET",
params,
expected_code=200,
headers=headers,
)
manifest_list = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(test_params["manifest_json"]),
test_params["manifest_type"],
sparse_manifest_support=True,
)
try:
manifest = Manifest.filter(
Manifest.digest == manifest_list.digest).get()
except Manifest.DoesNotExist:
assert False, "failed to create manifest list"
input_digests = [
manifest["digest"]
for manifest in manifest_list.manifest_dict["manifests"]
]
manifest_links = ManifestChild.select(
ManifestChild.child_manifest).where(
ManifestChild.manifest == manifest)
sub_digests = [ml.child_manifest.digest for ml in manifest_links]
assert input_digests == sub_digests
for link in manifest_links:
mbytes = link.child_manifest.manifest_bytes
assert mbytes == "", f"child manifest bytes expected empty, but got {mbytes}"