def get_parsed_manifest(self, validate=True):
"""
Returns the parsed manifest for this manifest.
"""
assert self.internal_manifest_bytes
return parse_manifest_from_bytes(self.internal_manifest_bytes,
self.media_type,
validate=validate)
def _parse_manifest():
content_type = request.content_type or DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE
if content_type == "application/json":
# For back-compat.
content_type = DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE
try:
return parse_manifest_from_bytes(Bytes.for_string_or_unicode(request.data), content_type)
except ManifestException as me:
logger.exception("failed to parse manifest when writing by tagname")
raise ManifestInvalid(detail={"message": "failed to parse manifest: %s" % me})
def test_get_or_create_manifest_invalid_image(initialized_db):
repository = get_repository("devtable", "simple")
latest_tag = get_tag(repository, "latest")
manifest_bytes = Bytes.for_string_or_unicode(
latest_tag.manifest.manifest_bytes)
parsed = parse_manifest_from_bytes(manifest_bytes,
latest_tag.manifest.media_type.name,
validate=False)
builder = DockerSchema1ManifestBuilder("devtable", "simple", "anothertag")
builder.add_layer(parsed.blob_digests[0],
'{"id": "foo", "parent": "someinvalidimageid"}')
sample_manifest_instance = builder.build(docker_v2_signing_key)
created_manifest = get_or_create_manifest(repository,
sample_manifest_instance,
storage)
assert created_manifest is None
def assert_gc_integrity(expect_storage_removed=True, check_oci_tags=True):
""" Specialized assertion for ensuring that GC cleans up all dangling storages
and labels, invokes the callback for images removed and doesn't invoke the
callback for images *not* removed.
"""
# Add a callback for when images are removed.
removed_image_storages = []
model.config.register_image_cleanup_callback(removed_image_storages.extend)
# Store the number of dangling storages and labels.
existing_storage_count = _get_dangling_storage_count()
existing_label_count = _get_dangling_label_count()
existing_manifest_count = _get_dangling_manifest_count()
yield
# Ensure the number of dangling storages, manifests and labels has not changed.
updated_storage_count = _get_dangling_storage_count()
assert updated_storage_count == existing_storage_count
updated_label_count = _get_dangling_label_count()
assert updated_label_count == existing_label_count, _get_dangling_labels()
updated_manifest_count = _get_dangling_manifest_count()
assert updated_manifest_count == existing_manifest_count
# Ensure that for each call to the image+storage cleanup callback, the image and its
# storage is not found *anywhere* in the database.
for removed_image_and_storage in removed_image_storages:
with pytest.raises(Image.DoesNotExist):
Image.get(id=removed_image_and_storage.id)
# Ensure that image storages are only removed if not shared.
shared = Image.select().where(
Image.storage == removed_image_and_storage.storage_id).count()
if shared == 0:
shared = (ManifestBlob.select().where(
ManifestBlob.blob ==
removed_image_and_storage.storage_id).count())
if shared == 0:
with pytest.raises(ImageStorage.DoesNotExist):
ImageStorage.get(id=removed_image_and_storage.storage_id)
with pytest.raises(ImageStorage.DoesNotExist):
ImageStorage.get(uuid=removed_image_and_storage.storage.uuid)
# Ensure all CAS storage is in the storage engine.
preferred = storage.preferred_locations[0]
for storage_row in ImageStorage.select():
if storage_row.cas_path:
storage.get_content({preferred},
storage.blob_path(
storage_row.content_checksum))
for blob_row in ApprBlob.select():
storage.get_content({preferred}, storage.blob_path(blob_row.digest))
# Ensure there are no danglings OCI tags.
if check_oci_tags:
oci_tags = {t.id for t in Tag.select()}
referenced_oci_tags = {t.tag_id for t in TagToRepositoryTag.select()}
assert not oci_tags - referenced_oci_tags
# Ensure all tags have valid manifests.
for manifest in {t.manifest for t in Tag.select()}:
# Ensure that the manifest's blobs all exist.
found_blobs = {
b.blob.content_checksum
for b in ManifestBlob.select().where(
ManifestBlob.manifest == manifest)
}
parsed = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(manifest.manifest_bytes),
manifest.media_type.name)
assert set(parsed.local_blob_digests) == found_blobs
def pull(
self,
session,
namespace,
repo_name,
tag_names,
images,
credentials=None,
expected_failure=None,
options=None,
):
options = options or ProtocolOptions()
scopes = options.scopes or ["repository:%s:pull" % self.repo_name(namespace, repo_name)]
tag_names = [tag_names] if isinstance(tag_names, str) else tag_names
# Ping!
self.ping(session)
# Perform auth and retrieve a token.
token, _ = self.auth(
session,
credentials,
namespace,
repo_name,
scopes=scopes,
expected_failure=expected_failure,
)
if token is None and not options.attempt_pull_without_token:
return None
headers = {}
if token:
headers = {
"Authorization": "Bearer " + token,
}
if self.schema2:
headers["Accept"] = ",".join(
options.accept_mimetypes
if options.accept_mimetypes is not None
else DOCKER_SCHEMA2_CONTENT_TYPES
)
manifests = {}
image_ids = {}
for tag_name in tag_names:
# Retrieve the manifest for the tag or digest.
response = self.conduct(
session,
"GET",
"/v2/%s/manifests/%s" % (self.repo_name(namespace, repo_name), tag_name),
expected_status=(200, expected_failure, V2ProtocolSteps.GET_MANIFEST),
headers=headers,
)
if response.status_code == 401:
assert "WWW-Authenticate" in response.headers
response.encoding = "utf-8"
if expected_failure is not None:
return None
# Ensure the manifest returned by us is valid.
ct = response.headers["Content-Type"]
if not self.schema2:
assert ct in DOCKER_SCHEMA1_CONTENT_TYPES
manifest = parse_manifest_from_bytes(Bytes.for_string_or_unicode(response.text), ct)
manifests[tag_name] = manifest
if manifest.schema_version == 1:
image_ids[tag_name] = manifest.leaf_layer_v1_image_id
# Verify the blobs.
layer_index = 0
empty_count = 0
blob_digests = list(manifest.blob_digests)
for image in images:
if manifest.schema_version == 2 and image.is_empty:
empty_count += 1
continue
# If the layer is remote, then we expect the blob to *not* exist in the system.
blob_digest = blob_digests[layer_index]
expected_status = 404 if image.urls else 200
result = self.conduct(
session,
"GET",
"/v2/%s/blobs/%s" % (self.repo_name(namespace, repo_name), blob_digest),
expected_status=(expected_status, expected_failure, V2ProtocolSteps.GET_BLOB),
headers=headers,
options=options,
)
if expected_status == 200:
assert result.content == image.bytes
layer_index += 1
assert (len(blob_digests) + empty_count) >= len(
images
) # Schema 2 has 1 extra for config
return PullResult(manifests=manifests, image_ids=image_ids)
def pull_list(
self,
session,
namespace,
repo_name,
tag_names,
manifestlist,
credentials=None,
expected_failure=None,
options=None,
):
options = options or ProtocolOptions()
scopes = options.scopes or [
"repository:%s:push,pull" % self.repo_name(namespace, repo_name)
]
tag_names = [tag_names] if isinstance(tag_names, str) else tag_names
# Ping!
self.ping(session)
# Perform auth and retrieve a token.
token, _ = self.auth(
session,
credentials,
namespace,
repo_name,
scopes=scopes,
expected_failure=expected_failure,
)
if token is None:
assert V2Protocol.FAILURE_CODES[V2ProtocolSteps.AUTH].get(expected_failure)
return
headers = {
"Authorization": "Bearer " + token,
"Accept": ",".join(DOCKER_SCHEMA2_CONTENT_TYPES),
}
for tag_name in tag_names:
# Retrieve the manifest for the tag or digest.
response = self.conduct(
session,
"GET",
"/v2/%s/manifests/%s" % (self.repo_name(namespace, repo_name), tag_name),
expected_status=(200, expected_failure, V2ProtocolSteps.GET_MANIFEST_LIST),
headers=headers,
)
if expected_failure is not None:
return None
# Parse the returned manifest list and ensure it matches.
ct = response.headers["Content-Type"]
assert ct == DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE
retrieved = parse_manifest_from_bytes(Bytes.for_string_or_unicode(response.text), ct)
assert retrieved.schema_version == 2
assert retrieved.is_manifest_list
assert retrieved.digest == manifestlist.digest
# Pull each of the manifests inside and ensure they can be retrieved.
for manifest_digest in retrieved.child_manifest_digests():
response = self.conduct(
session,
"GET",
"/v2/%s/manifests/%s" % (self.repo_name(namespace, repo_name), manifest_digest),
expected_status=(200, expected_failure, V2ProtocolSteps.GET_MANIFEST),
headers=headers,
)
if expected_failure is not None:
return None
ct = response.headers["Content-Type"]
manifest = parse_manifest_from_bytes(Bytes.for_string_or_unicode(response.text), ct)
assert not manifest.is_manifest_list
assert manifest.digest == manifest_digest
def test_parse_manifest_from_bytes(media_type, manifest_bytes):
assert parse_manifest_from_bytes(
Bytes.for_string_or_unicode(manifest_bytes),
media_type,
validate=False)