def _parse_manifest():
content_type = request.content_type or DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE
if content_type == "application/json":
# For back-compat.
content_type = DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE
try:
return parse_manifest_from_bytes(
Bytes.for_string_or_unicode(request.data), content_type)
except ManifestException as me:
logger.exception("failed to parse manifest when writing by tagname")
raise ManifestInvalid(
detail={"message": "failed to parse manifest: %s" % me})
def test_renew_manifest_and_parent_tag_when_manifest_is_child_of_manifest_list(
self, create_repo, proxy_manifest_response):
repo_ref = create_repo(self.orgname, self.upstream_repository,
self.user)
input_list = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(UBI8_LATEST_MANIFEST_LIST),
DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE,
sparse_manifest_support=True,
)
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock()):
proxy_model = ProxyModel(
self.orgname,
self.upstream_repository,
self.user,
)
manifest_list, tag = proxy_model._create_manifest_and_retarget_tag(
repo_ref, input_list, "latest")
assert manifest_list is not None
child = (ManifestChild.select(ManifestChild.child_manifest_id).join(
Manifest,
on=(ManifestChild.child_manifest_id == Manifest.id
)).where((ManifestChild.manifest_id == manifest_list.id)
& (Manifest.digest == UBI8_LATEST_DIGEST)))
manifest_tag = Tag.select().where(Tag.manifest == child).get()
manifest_list_tag = tag
proxy_mock = proxy_manifest_response(
UBI8_LATEST_DIGEST, UBI8_LATEST_MANIFEST_SCHEMA2,
DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE)
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
proxy_model = ProxyModel(
self.orgname,
self.upstream_repository,
self.user,
)
manifest = proxy_model.lookup_manifest_by_digest(
repo_ref, UBI8_LATEST_DIGEST)
updated_tag = oci.tag.get_tag_by_manifest_id(repo_ref.id, manifest.id)
updated_list_tag = oci.tag.get_tag_by_manifest_id(
repo_ref.id, manifest_list.id)
assert updated_tag.id == manifest_tag.id
assert updated_list_tag.id == manifest_list_tag.id
assert updated_tag.lifetime_end_ms > manifest_tag.lifetime_end_ms
assert updated_list_tag.lifetime_end_ms > manifest_list_tag.lifetime_end_ms
def test_does_not_pull_from_upstream_when_manifest_already_exists(
self, test_name, proxy_manifest_response):
test_params = storage_test_cases[test_name]
repo = f"{self.orgname}/{test_params['image_name']}"
params = {
"repository": repo,
"manifest_ref": test_params["manifest_ref"],
}
r = model.repository.create_repository(self.orgname,
test_params["image_name"],
self.user)
assert r is not None
manifest = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(test_params["manifest_json"]),
test_params["manifest_type"],
sparse_manifest_support=True,
)
m = oci.manifest.create_manifest(r.id, manifest)
assert m is not None
if test_params["ref_type"] == "digest":
oci.tag.create_temporary_tag_if_necessary(m, 300)
else:
oci.tag.retarget_tag(test_params["manifest_ref"], m.id)
proxy_mock = proxy_manifest_response(
test_params["manifest_ref"],
test_params["manifest_json"],
test_params["manifest_type"],
)
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
headers = _get_auth_headers(self.sub, self.ctx, repo)
headers["Accept"] = ", ".join(
DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union(
DOCKER_SCHEMA1_CONTENT_TYPES))
conduct_call(
self.client,
test_params["view_name"],
url_for,
"GET",
params,
expected_code=200,
headers=headers,
)
assert proxy_mock.manifest_exists.call_count == 1
assert proxy_mock.get_manifest.call_count == 0
def test_raises_exception_with_manifest_list(self):
manifest = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(UBI8_LATEST_MANIFEST_LIST),
DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE,
)
proxy_model = ProxyModel(
self.orgname,
self.upstream_repository,
self.user,
)
with pytest.raises(ManifestException):
proxy_model.get_schema1_parsed_manifest(
manifest,
self.orgname,
self.upstream_repository,
self.tag,
storage,
raise_on_error=True,
)
def test_create_manifest_and_temp_tag_when_they_dont_exist(
self, create_repo):
repo_ref = create_repo(self.orgname, self.upstream_repository,
self.user)
input_manifest = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(UBI8_8_4_MANIFEST_SCHEMA2),
DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE,
)
proxy_model = ProxyModel(
self.orgname,
self.upstream_repository,
self.user,
)
manifest, tag = proxy_model._create_manifest_and_retarget_tag(
repo_ref, input_manifest, self.tag)
assert manifest is not None
assert tag is not None
assert manifest.internal_manifest_bytes.as_unicode(
) == UBI8_8_4_MANIFEST_SCHEMA2
assert manifest.digest == UBI8_8_4_DIGEST
def test_get_or_create_manifest_invalid_image(initialized_db):
repository = get_repository("devtable", "simple")
latest_tag = get_tag(repository, "latest")
manifest_bytes = Bytes.for_string_or_unicode(
latest_tag.manifest.manifest_bytes)
parsed = parse_manifest_from_bytes(manifest_bytes,
latest_tag.manifest.media_type.name,
validate=False)
builder = DockerSchema1ManifestBuilder("devtable", "simple", "anothertag")
builder.add_layer(parsed.blob_digests[0],
'{"id": "foo", "parent": "someinvalidimageid"}')
sample_manifest_instance = builder.build(docker_v2_signing_key)
created_manifest = get_or_create_manifest(repository,
sample_manifest_instance,
storage)
assert created_manifest is None
def _pull_upstream_manifest(self, repo: str,
manifest_ref: str) -> ManifestInterface:
try:
raw_manifest, content_type = self._proxy.get_manifest(
manifest_ref, ACCEPTED_MEDIA_TYPES)
except UpstreamRegistryError as e:
raise ManifestDoesNotExist(str(e))
upstream_repo_name = self._upstream_repo(repo)
upstream_namespace = self._upstream_namespace(repo)
# TODO: do we need the compatibility check from v2._parse_manifest?
mbytes = Bytes.for_string_or_unicode(raw_manifest)
manifest = parse_manifest_from_bytes(mbytes,
content_type,
sparse_manifest_support=True)
valid = self._validate_schema1_manifest(upstream_namespace,
upstream_repo_name, manifest)
if not valid:
raise ManifestDoesNotExist("invalid schema 1 manifest")
return manifest
def test_create_sub_manifests_for_manifest_list(self, create_repo):
repo_ref = create_repo(self.orgname, self.upstream_repository,
self.user)
input_manifest = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(UBI8_LATEST_MANIFEST_LIST),
DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE,
sparse_manifest_support=True,
)
proxy_model = ProxyModel(
self.orgname,
self.upstream_repository,
self.user,
)
manifest, _ = proxy_model._create_manifest_and_retarget_tag(
repo_ref, input_manifest, self.tag)
mchildren = ManifestChild.select().where(
ManifestChild.manifest == manifest.id)
created_count = mchildren.count()
expected_count = len(
list(input_manifest.child_manifests(content_retriever=None)))
assert expected_count == created_count
def test_create_manifest_config_blob(self, test_name,
proxy_manifest_response):
test_params = storage_test_cases[test_name]
repo = f"{self.orgname}/{test_params['image_name']}"
params = {
"repository": repo,
"manifest_ref": test_params["manifest_ref"],
}
proxy_mock = proxy_manifest_response(
test_params["manifest_ref"],
test_params["manifest_json"],
test_params["manifest_type"],
)
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
headers = _get_auth_headers(self.sub, self.ctx, repo)
headers["Accept"] = ", ".join(
DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union(
DOCKER_SCHEMA1_CONTENT_TYPES))
resp = conduct_call(
self.client,
test_params["view_name"],
url_for,
"GET",
params,
expected_code=200,
headers=headers,
)
manifest = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(test_params["manifest_json"]),
test_params["manifest_type"],
sparse_manifest_support=True,
)
if manifest.schema_version == 2 and not manifest.is_manifest_list:
q = ImageStorage.filter(
ImageStorage.content_checksum == manifest.config.digest)
assert q.count() == 1
def test_connect_existing_blobs_to_new_manifest(self, create_repo):
repo_ref = create_repo(self.orgname, self.upstream_repository,
self.user)
input_manifest = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(UBI8_8_4_MANIFEST_SCHEMA2),
DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE,
)
layer = input_manifest.manifest_dict["layers"][0]
blob = ImageStorage.create(
image_size=layer["size"],
uncompressed_size=layer["size"],
content_checksum=layer["digest"],
)
proxy_model = ProxyModel(
self.orgname,
self.upstream_repository,
self.user,
)
proxy_model._create_manifest_and_retarget_tag(repo_ref, input_manifest,
self.tag)
blob_count = (ImageStorage.select().where(
ImageStorage.content_checksum == blob.content_checksum).count())
assert blob_count == 1
def assert_gc_integrity(expect_storage_removed=True):
"""
Specialized assertion for ensuring that GC cleans up all dangling storages and labels, invokes
the callback for images removed and doesn't invoke the callback for images *not* removed.
"""
# Add a callback for when images are removed.
removed_image_storages = []
remove_callback = model.config.register_image_cleanup_callback(
removed_image_storages.extend)
# Store existing storages. We won't verify these for existence because they
# were likely created as test data.
existing_digests = set()
for storage_row in ImageStorage.select():
if storage_row.cas_path:
existing_digests.add(storage_row.content_checksum)
for blob_row in ApprBlob.select():
existing_digests.add(blob_row.digest)
# Store the number of dangling objects.
existing_storage_count = _get_dangling_storage_count()
existing_label_count = _get_dangling_label_count()
existing_manifest_count = _get_dangling_manifest_count()
# Yield to the GC test.
with check_transitive_modifications():
try:
yield
finally:
remove_callback()
# Ensure the number of dangling storages, manifests and labels has not changed.
updated_storage_count = _get_dangling_storage_count()
assert updated_storage_count == existing_storage_count
updated_label_count = _get_dangling_label_count()
assert updated_label_count == existing_label_count, _get_dangling_labels()
updated_manifest_count = _get_dangling_manifest_count()
assert updated_manifest_count == existing_manifest_count
# Ensure that for each call to the image+storage cleanup callback, the image and its
# storage is not found *anywhere* in the database.
for removed_image_and_storage in removed_image_storages:
assert isinstance(removed_image_and_storage, Image)
try:
# NOTE: SQLite can and will reuse AUTOINCREMENT IDs occasionally, so if we find a row
# with the same ID, make sure it does not have the same Docker Image ID.
# See: https://www.sqlite.org/autoinc.html
found_image = Image.get(id=removed_image_and_storage.id)
assert (found_image.docker_image_id !=
removed_image_and_storage.docker_image_id
), "Found unexpected removed image %s under repo %s" % (
found_image.id,
found_image.repository,
)
except Image.DoesNotExist:
pass
# Ensure that image storages are only removed if not shared.
shared = Image.select().where(
Image.storage == removed_image_and_storage.storage_id).count()
if shared == 0:
shared = (ManifestBlob.select().where(
ManifestBlob.blob ==
removed_image_and_storage.storage_id).count())
if shared == 0:
shared = (UploadedBlob.select().where(
UploadedBlob.blob ==
removed_image_and_storage.storage_id).count())
if shared == 0:
with pytest.raises(ImageStorage.DoesNotExist):
ImageStorage.get(id=removed_image_and_storage.storage_id)
with pytest.raises(ImageStorage.DoesNotExist):
ImageStorage.get(uuid=removed_image_and_storage.storage.uuid)
# Ensure all CAS storage is in the storage engine.
preferred = storage.preferred_locations[0]
for storage_row in ImageStorage.select():
if storage_row.content_checksum in existing_digests:
continue
if storage_row.cas_path:
storage.get_content({preferred},
storage.blob_path(
storage_row.content_checksum))
for blob_row in ApprBlob.select():
if blob_row.digest in existing_digests:
continue
storage.get_content({preferred}, storage.blob_path(blob_row.digest))
# Ensure all tags have valid manifests.
for manifest in {t.manifest for t in Tag.select()}:
# Ensure that the manifest's blobs all exist.
found_blobs = {
b.blob.content_checksum
for b in ManifestBlob.select().where(
ManifestBlob.manifest == manifest)
}
parsed = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(manifest.manifest_bytes),
manifest.media_type.name)
assert set(parsed.local_blob_digests) == found_blobs
def pull_list(
self,
session,
namespace,
repo_name,
tag_names,
manifestlist,
credentials=None,
expected_failure=None,
options=None,
):
options = options or ProtocolOptions()
scopes = options.scopes or [
"repository:%s:push,pull" % self.repo_name(namespace, repo_name)
]
tag_names = [tag_names] if isinstance(tag_names, str) else tag_names
# Ping!
self.ping(session)
# Perform auth and retrieve a token.
token, _ = self.auth(
session,
credentials,
namespace,
repo_name,
scopes=scopes,
expected_failure=expected_failure,
)
if token is None:
assert V2Protocol.FAILURE_CODES[V2ProtocolSteps.AUTH].get(expected_failure)
return
headers = {
"Authorization": "Bearer " + token,
"Accept": ",".join(MANIFEST_LIST_TYPES),
}
for tag_name in tag_names:
# Retrieve the manifest for the tag or digest.
response = self.conduct(
session,
"GET",
"/v2/%s/manifests/%s" % (self.repo_name(namespace, repo_name), tag_name),
expected_status=(200, expected_failure, V2ProtocolSteps.GET_MANIFEST_LIST),
headers=headers,
)
if expected_failure is not None:
return None
# Parse the returned manifest list and ensure it matches.
ct = response.headers["Content-Type"]
assert is_manifest_list_type(ct), "Expected list type, found: %s" % ct
retrieved = parse_manifest_from_bytes(Bytes.for_string_or_unicode(response.text), ct)
assert retrieved.schema_version == 2
assert retrieved.is_manifest_list
assert retrieved.digest == manifestlist.digest
# Pull each of the manifests inside and ensure they can be retrieved.
for manifest_digest in retrieved.child_manifest_digests():
response = self.conduct(
session,
"GET",
"/v2/%s/manifests/%s" % (self.repo_name(namespace, repo_name), manifest_digest),
expected_status=(200, expected_failure, V2ProtocolSteps.GET_MANIFEST),
headers=headers,
)
if expected_failure is not None:
return None
ct = response.headers["Content-Type"]
manifest = parse_manifest_from_bytes(Bytes.for_string_or_unicode(response.text), ct)
assert not manifest.is_manifest_list
assert manifest.digest == manifest_digest
def pull(
self,
session,
namespace,
repo_name,
tag_names,
images,
credentials=None,
expected_failure=None,
options=None,
):
options = options or ProtocolOptions()
scopes = options.scopes or ["repository:%s:pull" % self.repo_name(namespace, repo_name)]
tag_names = [tag_names] if isinstance(tag_names, str) else tag_names
# Ping!
self.ping(session)
# Perform auth and retrieve a token.
token, _ = self.auth(
session,
credentials,
namespace,
repo_name,
scopes=scopes,
expected_failure=expected_failure,
)
if token is None and not options.attempt_pull_without_token:
return None
headers = {}
if token:
headers = {
"Authorization": "Bearer " + token,
}
if self.schema == "oci":
headers["Accept"] = ",".join(
options.accept_mimetypes
if options.accept_mimetypes is not None
else OCI_CONTENT_TYPES
)
elif self.schema == "schema2":
headers["Accept"] = ",".join(
options.accept_mimetypes
if options.accept_mimetypes is not None
else DOCKER_SCHEMA2_CONTENT_TYPES
)
manifests = {}
image_ids = {}
for tag_name in tag_names:
# Retrieve the manifest for the tag or digest.
response = self.conduct(
session,
"GET",
"/v2/%s/manifests/%s" % (self.repo_name(namespace, repo_name), tag_name),
expected_status=(200, expected_failure, V2ProtocolSteps.GET_MANIFEST),
headers=headers,
)
if response.status_code == 401:
assert "WWW-Authenticate" in response.headers
response.encoding = "utf-8"
if expected_failure is not None:
return None
# Ensure the manifest returned by us is valid.
ct = response.headers["Content-Type"]
if self.schema == "schema1":
assert ct in DOCKER_SCHEMA1_CONTENT_TYPES
if options.require_matching_manifest_type:
if self.schema == "schema1":
assert ct in DOCKER_SCHEMA1_CONTENT_TYPES
if self.schema == "schema2":
assert ct in DOCKER_SCHEMA2_CONTENT_TYPES
if self.schema == "oci":
assert ct in OCI_CONTENT_TYPES
manifest = parse_manifest_from_bytes(Bytes.for_string_or_unicode(response.text), ct)
manifests[tag_name] = manifest
if manifest.schema_version == 1:
image_ids[tag_name] = manifest.leaf_layer_v1_image_id
# Verify the blobs.
layer_index = 0
empty_count = 0
blob_digests = list(manifest.blob_digests)
for image in images:
if manifest.schema_version == 2 and image.is_empty:
empty_count += 1
continue
# If the layer is remote, then we expect the blob to *not* exist in the system.
blob_digest = blob_digests[layer_index]
expected_status = 404 if image.urls else 200
result = self.conduct(
session,
"GET",
"/v2/%s/blobs/%s" % (self.repo_name(namespace, repo_name), blob_digest),
expected_status=(expected_status, expected_failure, V2ProtocolSteps.GET_BLOB),
headers=headers,
options=options,
)
if expected_status == 200:
assert result.content == image.bytes
layer_index += 1
assert (len(blob_digests) + empty_count) >= len(
images
) # OCI/Schema 2 has 1 extra for config
return PullResult(manifests=manifests, image_ids=image_ids)
def test_manifest_list_create_manifest_with_sub_manifests_and_connect_them(
self, test_name, proxy_manifest_response):
test_params = storage_test_cases[test_name]
if test_params["manifest_type"] not in [
DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE,
OCI_IMAGE_INDEX_CONTENT_TYPE,
]:
pytest.skip(
"regular manifest detected, skipping manifest list specific test."
)
repo = f"{self.orgname}/{test_params['image_name']}"
params = {
"repository": repo,
"manifest_ref": test_params["manifest_ref"],
}
proxy_mock = proxy_manifest_response(
test_params["manifest_ref"],
test_params["manifest_json"],
test_params["manifest_type"],
)
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
headers = _get_auth_headers(self.sub, self.ctx, repo)
headers["Accept"] = ", ".join(
DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union(
DOCKER_SCHEMA1_CONTENT_TYPES))
conduct_call(
self.client,
test_params["view_name"],
url_for,
"GET",
params,
expected_code=200,
headers=headers,
)
manifest_list = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(test_params["manifest_json"]),
test_params["manifest_type"],
sparse_manifest_support=True,
)
try:
manifest = Manifest.filter(
Manifest.digest == manifest_list.digest).get()
except Manifest.DoesNotExist:
assert False, "failed to create manifest list"
input_digests = [
manifest["digest"]
for manifest in manifest_list.manifest_dict["manifests"]
]
manifest_links = ManifestChild.select(
ManifestChild.child_manifest).where(
ManifestChild.manifest == manifest)
sub_digests = [ml.child_manifest.digest for ml in manifest_links]
assert input_digests == sub_digests
for link in manifest_links:
mbytes = link.child_manifest.manifest_bytes
assert mbytes == "", f"child manifest bytes expected empty, but got {mbytes}"
def test_pull_placeholder_manifest_updates_manifest_bytes(
self, test_name, proxy_manifest_response):
"""
it's not possible to connect a sub-manifest to a manifest list on a subsequent pull,
since a regular manifest request has no pointer to the manifest list it belongs to.
to connect the sub-manifests with the manifest list being at pull time, we create a
placeholder manifest.
placeholder manifests are caracterized by having an empty manifest_bytes.
"""
test_params = storage_test_cases[test_name]
if test_params["manifest_type"] in [
DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE,
OCI_IMAGE_INDEX_CONTENT_TYPE,
]:
pytest.skip(
"manifest list detected, skipping 'flat' manifest specific test."
)
# we only have the manifest list json for the hello-world
# (because it's significantly smaller).
if test_params["image_name"] == "busybox":
pytest.skip(
"skipping test for busybox image - we do not have its manifest list json."
)
if test_params["manifest_type"] == OCI_IMAGE_MANIFEST_CONTENT_TYPE:
pytest.skip(
"skipping OCI content type - manifest list specifies docker schema v2."
)
if test_params["ref_type"] == "tag":
pytest.skip(
"skipping manifest fetch by tag - pull for a specific architecture is made by digest",
)
parsed = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(HELLO_WORLD_MANIFEST_LIST_JSON),
DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE,
sparse_manifest_support=True,
)
# first create the manifest list and its placeholders
repo = f"{self.orgname}/{test_params['image_name']}"
params = {
"repository": repo,
"manifest_ref": parsed.digest,
}
proxy_mock = proxy_manifest_response(
parsed.digest,
HELLO_WORLD_MANIFEST_LIST_JSON,
DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE,
)
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
headers = _get_auth_headers(self.sub, self.ctx, repo)
headers["Accept"] = ", ".join(
DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union(
DOCKER_SCHEMA1_CONTENT_TYPES))
conduct_call(
self.client,
test_params["view_name"],
url_for,
"GET",
params,
expected_code=200,
headers=headers,
)
# now fetch one of the sub manifests from the manifest list
test_manifest = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(test_params["manifest_json"]),
test_params["manifest_type"],
sparse_manifest_support=True,
)
params = {
"repository": repo,
"manifest_ref": test_params["manifest_ref"],
}
proxy_mock = proxy_manifest_response(
test_params["manifest_ref"],
test_params["manifest_json"],
test_params["manifest_type"],
)
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
headers = _get_auth_headers(self.sub, self.ctx, repo)
headers["Accept"] = ", ".join(
DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union(
DOCKER_SCHEMA1_CONTENT_TYPES))
resp = conduct_call(
self.client,
test_params["view_name"],
url_for,
"GET",
params,
expected_code=200,
headers=headers,
)
sub_manifest = Manifest.filter(
Manifest.digest == test_params["manifest_ref"]).get()
assert sub_manifest.manifest_bytes != ""
output_manifest = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(sub_manifest.manifest_bytes),
sub_manifest.media_type.name,
sparse_manifest_support=True,
)
input_manifest = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(test_params["manifest_json"]),
test_params["manifest_type"],
sparse_manifest_support=True,
)
assert output_manifest.schema_version == input_manifest.schema_version
assert output_manifest.media_type == input_manifest.media_type
assert output_manifest.is_manifest_list == input_manifest.is_manifest_list
assert output_manifest.digest == input_manifest.digest
assert output_manifest.manifest_dict == input_manifest.manifest_dict
def test_parse_manifest_from_bytes(media_type, manifest_bytes):
assert parse_manifest_from_bytes(
Bytes.for_string_or_unicode(manifest_bytes),
media_type,
validate=False)
def test_create_placeholder_blobs_on_first_pull(self, test_name,
proxy_manifest_response):
test_params = storage_test_cases[test_name]
# no blob placeholders are created for manifest lists - we don't have
# the sub-manifests at manifest list creation time, so there's no way
# to know which blobs the sub-manifest has.
if test_params["manifest_type"] in [
DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE,
OCI_IMAGE_INDEX_CONTENT_TYPE,
]:
pytest.skip(
"manifest list detected - skipping blob placeholder test")
repo = f"{self.orgname}/{test_params['image_name']}"
params = {
"repository": repo,
"manifest_ref": test_params["manifest_ref"],
}
proxy_mock = proxy_manifest_response(test_params["manifest_ref"],
test_params["manifest_json"],
test_params["manifest_type"])
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
headers = _get_auth_headers(self.sub, self.ctx, repo)
headers["Accept"] = ", ".join(
DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union(
DOCKER_SCHEMA1_CONTENT_TYPES))
conduct_call(
self.client,
test_params["view_name"],
url_for,
"GET",
params,
expected_code=200,
headers=headers,
)
parsed = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(test_params["manifest_json"]),
test_params["manifest_type"],
sparse_manifest_support=True,
)
manifest = Manifest.filter(Manifest.digest == parsed.digest).get()
mdict = parsed.manifest_dict
layers = mdict.get("layers", mdict.get("fsLayers"))
mblobs = ManifestBlob.filter(ManifestBlob.manifest == manifest)
expected_count = len(layers)
# schema 2 manifests have an extra config blob which we need to take into
# consideration in the total count
config_digest = ""
if parsed.schema_version == 2:
config_digest = parsed.config.digest
expected_count += 1
assert mblobs.count() == expected_count
for mblob in mblobs:
blob = None
layer = None
# don't assert if digest belongs to a config blob
if mblob.blob.content_checksum == config_digest:
continue
for layer in layers:
digest = layer.get("digest", layer.get("blobSum"))
if mblob.blob.content_checksum == digest:
blob = mblob.blob
layer = layer
break
assert blob is not None
assert blob.image_size == layer.get("size", None)
# the absence of an image storage placement for a blob indicates that it's
# a placeholder blob, not yet downloaded from the upstream registry.
placements = ImageStoragePlacement.filter(
ImageStoragePlacement.storage == blob)
assert placements.count() == 0
def assert_gc_integrity(expect_storage_removed=True):
"""
Specialized assertion for ensuring that GC cleans up all dangling storages and labels, invokes
the callback for images removed and doesn't invoke the callback for images *not* removed.
"""
# Add a callback for when images are removed.
removed_image_storages = []
model.config.register_image_cleanup_callback(removed_image_storages.extend)
# Store existing storages. We won't verify these for existence because they
# were likely created as test data.
existing_digests = set()
for storage_row in ImageStorage.select():
if storage_row.cas_path:
existing_digests.add(storage_row.content_checksum)
for blob_row in ApprBlob.select():
existing_digests.add(blob_row.digest)
# Store the number of dangling storages and labels.
existing_storage_count = _get_dangling_storage_count()
existing_label_count = _get_dangling_label_count()
existing_manifest_count = _get_dangling_manifest_count()
# Yield to the GC test.
with check_transitive_modifications():
yield
# Ensure the number of dangling storages, manifests and labels has not changed.
updated_storage_count = _get_dangling_storage_count()
assert updated_storage_count == existing_storage_count
updated_label_count = _get_dangling_label_count()
assert updated_label_count == existing_label_count, _get_dangling_labels()
updated_manifest_count = _get_dangling_manifest_count()
assert updated_manifest_count == existing_manifest_count
# Ensure that for each call to the image+storage cleanup callback, the image and its
# storage is not found *anywhere* in the database.
for removed_image_and_storage in removed_image_storages:
with pytest.raises(Image.DoesNotExist):
Image.get(id=removed_image_and_storage.id)
# Ensure that image storages are only removed if not shared.
shared = Image.select().where(
Image.storage == removed_image_and_storage.storage_id).count()
if shared == 0:
shared = (ManifestBlob.select().where(
ManifestBlob.blob ==
removed_image_and_storage.storage_id).count())
if shared == 0:
with pytest.raises(ImageStorage.DoesNotExist):
ImageStorage.get(id=removed_image_and_storage.storage_id)
with pytest.raises(ImageStorage.DoesNotExist):
ImageStorage.get(uuid=removed_image_and_storage.storage.uuid)
# Ensure all CAS storage is in the storage engine.
preferred = storage.preferred_locations[0]
for storage_row in ImageStorage.select():
if storage_row.content_checksum in existing_digests:
continue
if storage_row.cas_path:
storage.get_content({preferred},
storage.blob_path(
storage_row.content_checksum))
for blob_row in ApprBlob.select():
if blob_row.digest in existing_digests:
continue
storage.get_content({preferred}, storage.blob_path(blob_row.digest))
# Ensure all tags have valid manifests.
for manifest in {t.manifest for t in Tag.select()}:
# Ensure that the manifest's blobs all exist.
found_blobs = {
b.blob.content_checksum
for b in ManifestBlob.select().where(
ManifestBlob.manifest == manifest)
}
parsed = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(manifest.manifest_bytes),
manifest.media_type.name)
assert set(parsed.local_blob_digests) == found_blobs
