def build(self, ensure_ascii=True):
"""
Builds and returns the DockerSchema2Manifest.
"""
assert self.config
def _build_layer(layer):
if layer.urls:
return {
DOCKER_SCHEMA2_MANIFEST_MEDIATYPE_KEY: DOCKER_SCHEMA2_REMOTE_LAYER_CONTENT_TYPE,
DOCKER_SCHEMA2_MANIFEST_SIZE_KEY: layer.compressed_size,
DOCKER_SCHEMA2_MANIFEST_DIGEST_KEY: str(layer.digest),
DOCKER_SCHEMA2_MANIFEST_URLS_KEY: layer.urls,
}
return {
DOCKER_SCHEMA2_MANIFEST_MEDIATYPE_KEY: DOCKER_SCHEMA2_LAYER_CONTENT_TYPE,
DOCKER_SCHEMA2_MANIFEST_SIZE_KEY: layer.compressed_size,
DOCKER_SCHEMA2_MANIFEST_DIGEST_KEY: str(layer.digest),
}
manifest_dict = {
DOCKER_SCHEMA2_MANIFEST_VERSION_KEY: 2,
DOCKER_SCHEMA2_MANIFEST_MEDIATYPE_KEY: DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE,
# Config
DOCKER_SCHEMA2_MANIFEST_CONFIG_KEY: {
DOCKER_SCHEMA2_MANIFEST_MEDIATYPE_KEY: DOCKER_SCHEMA2_CONFIG_CONTENT_TYPE,
DOCKER_SCHEMA2_MANIFEST_SIZE_KEY: self.config.size,
DOCKER_SCHEMA2_MANIFEST_DIGEST_KEY: str(self.config.digest),
},
# Layers
DOCKER_SCHEMA2_MANIFEST_LAYERS_KEY: [
_build_layer(layer) for layer in self.filesystem_layers
],
}
json_str = json.dumps(manifest_dict, ensure_ascii=ensure_ascii, indent=3)
return DockerSchema2Manifest(Bytes.for_string_or_unicode(json_str))
def test_create_manifest_config_blob(self, test_name,
proxy_manifest_response):
test_params = storage_test_cases[test_name]
repo = f"{self.orgname}/{test_params['image_name']}"
params = {
"repository": repo,
"manifest_ref": test_params["manifest_ref"],
}
proxy_mock = proxy_manifest_response(
test_params["manifest_ref"],
test_params["manifest_json"],
test_params["manifest_type"],
)
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
headers = _get_auth_headers(self.sub, self.ctx, repo)
headers["Accept"] = ", ".join(
DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union(
DOCKER_SCHEMA1_CONTENT_TYPES))
resp = conduct_call(
self.client,
test_params["view_name"],
url_for,
"GET",
params,
expected_code=200,
headers=headers,
)
manifest = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(test_params["manifest_json"]),
test_params["manifest_type"],
sparse_manifest_support=True,
)
if manifest.schema_version == 2 and not manifest.is_manifest_list:
q = ImageStorage.filter(
ImageStorage.content_checksum == manifest.config.digest)
assert q.count() == 1
def for_manifest(cls, manifest, legacy_id_handler, legacy_image_row=None):
if manifest is None:
return None
# NOTE: `manifest_bytes` will be None if not selected by certain join queries.
manifest_bytes = (
Bytes.for_string_or_unicode(manifest.manifest_bytes)
if manifest.manifest_bytes is not None
else None
)
return Manifest(
db_id=manifest.id,
digest=manifest.digest,
internal_manifest_bytes=manifest_bytes,
media_type=ManifestTable.media_type.get_name(manifest.media_type_id),
_layers_compressed_size=manifest.layers_compressed_size,
config_media_type=manifest.config_media_type,
inputs=dict(
legacy_id_handler=legacy_id_handler,
legacy_image_row=legacy_image_row,
repository=RepositoryReference.for_id(manifest.repository_id),
),
)
def _validate(self):
if not self._signatures:
return
payload_str = self._payload
for signature in self._signatures:
bytes_to_verify = b"%s.%s" % (
Bytes.for_string_or_unicode(
signature["protected"]).as_encoded_str(),
base64url_encode(payload_str),
)
signer = SIGNER_ALGS[signature["header"]["alg"]]
key = keyrep(signature["header"]["jwk"])
gk = key.get_key()
sig = base64url_decode(signature["signature"].encode("utf-8"))
try:
verified = signer.verify(bytes_to_verify, sig, gk)
except BadSignature:
raise InvalidSchema1Signature()
if not verified:
raise InvalidSchema1Signature()
def build(self, ensure_ascii=True):
"""
Builds and returns the OCIManifest.
"""
assert self.filesystem_layers
assert self.config
def _build_layer(layer):
if layer.urls:
return {
OCI_MANIFEST_MEDIATYPE_KEY: OCI_IMAGE_TAR_GZIP_NON_DISTRIBUTABLE_LAYER_CONTENT_TYPE,
OCI_MANIFEST_SIZE_KEY: layer.compressed_size,
OCI_MANIFEST_DIGEST_KEY: str(layer.digest),
OCI_MANIFEST_URLS_KEY: layer.urls,
}
return {
OCI_MANIFEST_MEDIATYPE_KEY: OCI_IMAGE_TAR_GZIP_LAYER_CONTENT_TYPE,
OCI_MANIFEST_SIZE_KEY: layer.compressed_size,
OCI_MANIFEST_DIGEST_KEY: str(layer.digest),
}
manifest_dict = {
OCI_MANIFEST_VERSION_KEY: 2,
OCI_MANIFEST_MEDIATYPE_KEY: OCI_IMAGE_MANIFEST_CONTENT_TYPE,
# Config
OCI_MANIFEST_CONFIG_KEY: {
OCI_MANIFEST_MEDIATYPE_KEY: OCI_IMAGE_CONFIG_CONTENT_TYPE,
OCI_MANIFEST_SIZE_KEY: self.config.size,
OCI_MANIFEST_DIGEST_KEY: str(self.config.digest),
},
# Layers
OCI_MANIFEST_LAYERS_KEY: [_build_layer(layer) for layer in self.filesystem_layers],
}
json_str = json.dumps(manifest_dict, ensure_ascii=ensure_ascii, indent=3)
return OCIManifest(Bytes.for_string_or_unicode(json_str))
def test_valid_manifest():
manifest = DockerSchema1Manifest(Bytes.for_string_or_unicode(MANIFEST_BYTES), validate=False)
assert len(manifest.signatures) == 1
assert manifest.namespace == ""
assert manifest.repo_name == "hello-world"
assert manifest.tag == "latest"
assert manifest.image_ids == {"someid", "anotherid"}
assert manifest.parent_image_ids == {"anotherid"}
assert len(manifest.layers) == 2
assert manifest.layers[0].v1_metadata.image_id == "anotherid"
assert manifest.layers[0].v1_metadata.parent_image_id is None
assert manifest.layers[1].v1_metadata.image_id == "someid"
assert manifest.layers[1].v1_metadata.parent_image_id == "anotherid"
assert manifest.layers[0].compressed_size is None
assert manifest.layers[1].compressed_size is None
assert manifest.leaf_layer == manifest.layers[1]
assert manifest.created_datetime is None
unsigned = manifest.unsigned()
assert unsigned.namespace == manifest.namespace
assert unsigned.repo_name == manifest.repo_name
assert unsigned.tag == manifest.tag
assert unsigned.layers == manifest.layers
assert unsigned.blob_digests == manifest.blob_digests
assert unsigned.digest != manifest.digest
image_layers = list(manifest.get_layers(None))
assert len(image_layers) == 2
for index in range(0, 2):
assert image_layers[index].layer_id == manifest.layers[index].v1_metadata.image_id
assert image_layers[index].blob_digest == manifest.layers[index].digest
assert image_layers[index].command == manifest.layers[index].v1_metadata.command
def test_validate_manifest_without_metadata():
test_dir = os.path.dirname(os.path.abspath(__file__))
with open(os.path.join(test_dir, "validated_manifest.json"), "r") as f:
manifest_bytes = f.read()
manifest = DockerSchema1Manifest(Bytes.for_string_or_unicode(manifest_bytes), validate=True)
digest = manifest.digest
assert digest == "sha256:b5dc4f63fdbd64f34f2314c0747ef81008f9fcddce4edfc3fd0e8ec8b358d571"
assert manifest.created_datetime
with_metadata_removed = manifest._unsigned_builder().with_metadata_removed().build()
assert with_metadata_removed.leaf_layer_v1_image_id == manifest.leaf_layer_v1_image_id
manifest_layers = list(manifest.get_layers(None))
with_metadata_removed_layers = list(with_metadata_removed.get_layers(None))
assert len(manifest_layers) == len(with_metadata_removed_layers)
for index, built_layer in enumerate(manifest_layers):
with_metadata_removed_layer = with_metadata_removed_layers[index]
assert built_layer.layer_id == with_metadata_removed_layer.layer_id
assert built_layer.compressed_size == with_metadata_removed_layer.compressed_size
assert built_layer.command == with_metadata_removed_layer.command
assert built_layer.comment == with_metadata_removed_layer.comment
assert built_layer.author == with_metadata_removed_layer.author
assert built_layer.blob_digest == with_metadata_removed_layer.blob_digest
assert built_layer.created_datetime == with_metadata_removed_layer.created_datetime
assert with_metadata_removed.digest != manifest.digest
assert with_metadata_removed.namespace == manifest.namespace
assert with_metadata_removed.repo_name == manifest.repo_name
assert with_metadata_removed.tag == manifest.tag
assert with_metadata_removed.created_datetime == manifest.created_datetime
assert with_metadata_removed.checksums == manifest.checksums
assert with_metadata_removed.image_ids == manifest.image_ids
assert with_metadata_removed.parent_image_ids == manifest.parent_image_ids
def test_valid_manifestlist():
manifestlist = DockerSchema2ManifestList(
Bytes.for_string_or_unicode(MANIFESTLIST_BYTES))
assert len(manifestlist.manifests(retriever)) == 2
assert manifestlist.media_type == "application/vnd.docker.distribution.manifest.list.v2+json"
assert manifestlist.bytes.as_encoded_str() == MANIFESTLIST_BYTES
assert manifestlist.manifest_dict == json.loads(MANIFESTLIST_BYTES)
assert manifestlist.get_layers(retriever) is None
assert not manifestlist.blob_digests
for index, manifest in enumerate(manifestlist.manifests(retriever)):
if index == 0:
assert isinstance(manifest.manifest_obj, DockerSchema2Manifest)
assert manifest.manifest_obj.schema_version == 2
else:
assert isinstance(manifest.manifest_obj, DockerSchema1Manifest)
assert manifest.manifest_obj.schema_version == 1
# Check retrieval of a schema 2 manifest. This should return None, because the schema 2 manifest
# is not amd64-compatible.
schema2_manifest = manifestlist.convert_manifest(
[DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE], "foo", "bar", "baz", retriever)
assert schema2_manifest is None
# Check retrieval of a schema 1 manifest.
compatible_manifest = manifestlist.get_schema1_manifest(
"foo", "bar", "baz", retriever)
assert compatible_manifest.schema_version == 1
schema1_manifest = manifestlist.convert_manifest(
DOCKER_SCHEMA1_CONTENT_TYPES, "foo", "bar", "baz", retriever)
assert schema1_manifest.schema_version == 1
assert schema1_manifest.digest == compatible_manifest.digest
# Ensure it validates.
manifestlist.validate(retriever)
def test_get_schema1_manifest():
retriever = ContentRetrieverForTesting.for_config(
{
"config": {"Labels": {"foo": "bar",},},
"rootfs": {"type": "layers", "diff_ids": []},
"history": [
{"created": "2018-04-03T18:37:09.284840891Z", "created_by": "foo"},
{"created": "2018-04-12T18:37:09.284840891Z", "created_by": "bar"},
{"created": "2018-04-03T18:37:09.284840891Z", "created_by": "foo"},
],
"architecture": "amd64",
"os": "linux",
},
"sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7",
7023,
)
manifest = OCIManifest(Bytes.for_string_or_unicode(SAMPLE_MANIFEST))
assert manifest.get_manifest_labels(retriever) == {
"com.example.key1": "value1",
"com.example.key2": "value2",
"foo": "bar",
}
schema1 = manifest.get_schema1_manifest("somenamespace", "somename", "sometag", retriever)
assert schema1 is not None
assert schema1.media_type == DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE
assert set(schema1.local_blob_digests) == (
set(manifest.local_blob_digests)
- {"sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7"}
)
assert len(schema1.layers) == 3
via_convert = manifest.convert_manifest(
[schema1.media_type], "somenamespace", "somename", "sometag", retriever
)
assert via_convert.digest == schema1.digest
def test_connect_existing_blobs_to_new_manifest(self, create_repo):
repo_ref = create_repo(self.orgname, self.upstream_repository,
self.user)
input_manifest = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(UBI8_8_4_MANIFEST_SCHEMA2),
DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE,
)
layer = input_manifest.manifest_dict["layers"][0]
blob = ImageStorage.create(
image_size=layer["size"],
uncompressed_size=layer["size"],
content_checksum=layer["digest"],
)
proxy_model = ProxyModel(
self.orgname,
self.upstream_repository,
self.user,
)
proxy_model._create_manifest_and_retarget_tag(repo_ref, input_manifest,
self.tag)
blob_count = (ImageStorage.select().where(
ImageStorage.content_checksum == blob.content_checksum).count())
assert blob_count == 1
def access_valid_token(token_code):
"""
Looks up an unexpired app specific token with the given token code.
If found, the token's last_accessed field is set to now and the token is returned. If not found,
returns None.
"""
token_code = remove_unicode(
Bytes.for_string_or_unicode(token_code).as_encoded_str())
prefix = token_code[:TOKEN_NAME_PREFIX_LENGTH]
if len(prefix) != TOKEN_NAME_PREFIX_LENGTH:
return None
suffix = token_code[TOKEN_NAME_PREFIX_LENGTH:]
# Lookup the token by its prefix.
try:
token = (AppSpecificAuthToken.select(
AppSpecificAuthToken, User).join(User).where(
AppSpecificAuthToken.token_name == prefix,
((AppSpecificAuthToken.expiration > datetime.now())
| (AppSpecificAuthToken.expiration >> None)),
).get())
if not token.token_secret.matches(suffix):
return None
assert len(prefix) == TOKEN_NAME_PREFIX_LENGTH
assert len(suffix) >= MINIMUM_TOKEN_SUFFIX_LENGTH
update_last_accessed(token)
return token
except AppSpecificAuthToken.DoesNotExist:
pass
return None
def test_build_unencoded_unicode_manifest():
config_json = json.dumps(
{
"config": {
"author": "Sômé guy",
},
"rootfs": {"type": "layers", "diff_ids": []},
"history": [
{
"created": "2018-04-03T18:37:09.284840891Z",
"created_by": "base",
"author": "Sômé guy",
},
],
},
ensure_ascii=False,
)
schema2_config = DockerSchema2Config(Bytes.for_string_or_unicode(config_json))
builder = DockerSchema2ManifestBuilder()
builder.set_config(schema2_config)
builder.add_layer("sha256:abc123", 123)
builder.build()
def _load_manifest(self):
digest = self._manifest_data[self._digest_key]
size = self._manifest_data[self._size_key]
manifest_bytes = self._content_retriever.get_manifest_bytes_with_digest(
digest)
if manifest_bytes is None:
raise ManifestException(
"Could not find child manifest with digest `%s`" % digest)
if len(manifest_bytes) != size:
raise ManifestException(
"Size of manifest does not match that retrieved: %s vs %s",
len(manifest_bytes),
size,
)
content_type = self._manifest_data[self._media_type_key]
if content_type not in self._supported_types:
raise ManifestException(
"Unknown or unsupported manifest media type `%s`" %
content_type)
return self._supported_types[content_type](
Bytes.for_string_or_unicode(manifest_bytes), validate=False)
def build(self):
""" Builds and returns the DockerSchema2ManifestList. """
assert self.manifests
manifest_list_dict = {
DOCKER_SCHEMA2_MANIFESTLIST_VERSION_KEY:
2,
DOCKER_SCHEMA2_MANIFESTLIST_MEDIATYPE_KEY:
DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE,
DOCKER_SCHEMA2_MANIFESTLIST_MANIFESTS_KEY: [{
DOCKER_SCHEMA2_MANIFESTLIST_MEDIATYPE_KEY:
manifest[2],
DOCKER_SCHEMA2_MANIFESTLIST_DIGEST_KEY:
manifest[0],
DOCKER_SCHEMA2_MANIFESTLIST_SIZE_KEY:
manifest[1],
DOCKER_SCHEMA2_MANIFESTLIST_PLATFORM_KEY:
manifest[3],
} for manifest in self.manifests],
}
json_str = Bytes.for_string_or_unicode(
json.dumps(manifest_list_dict, indent=3))
return DockerSchema2ManifestList(json_str)
def assert_gc_integrity(expect_storage_removed=True, check_oci_tags=True):
""" Specialized assertion for ensuring that GC cleans up all dangling storages
and labels, invokes the callback for images removed and doesn't invoke the
callback for images *not* removed.
"""
# Add a callback for when images are removed.
removed_image_storages = []
model.config.register_image_cleanup_callback(removed_image_storages.extend)
# Store the number of dangling storages and labels.
existing_storage_count = _get_dangling_storage_count()
existing_label_count = _get_dangling_label_count()
existing_manifest_count = _get_dangling_manifest_count()
yield
# Ensure the number of dangling storages, manifests and labels has not changed.
updated_storage_count = _get_dangling_storage_count()
assert updated_storage_count == existing_storage_count
updated_label_count = _get_dangling_label_count()
assert updated_label_count == existing_label_count, _get_dangling_labels()
updated_manifest_count = _get_dangling_manifest_count()
assert updated_manifest_count == existing_manifest_count
# Ensure that for each call to the image+storage cleanup callback, the image and its
# storage is not found *anywhere* in the database.
for removed_image_and_storage in removed_image_storages:
with pytest.raises(Image.DoesNotExist):
Image.get(id=removed_image_and_storage.id)
# Ensure that image storages are only removed if not shared.
shared = Image.select().where(
Image.storage == removed_image_and_storage.storage_id).count()
if shared == 0:
shared = (ManifestBlob.select().where(
ManifestBlob.blob ==
removed_image_and_storage.storage_id).count())
if shared == 0:
with pytest.raises(ImageStorage.DoesNotExist):
ImageStorage.get(id=removed_image_and_storage.storage_id)
with pytest.raises(ImageStorage.DoesNotExist):
ImageStorage.get(uuid=removed_image_and_storage.storage.uuid)
# Ensure all CAS storage is in the storage engine.
preferred = storage.preferred_locations[0]
for storage_row in ImageStorage.select():
if storage_row.cas_path:
storage.get_content({preferred},
storage.blob_path(
storage_row.content_checksum))
for blob_row in ApprBlob.select():
storage.get_content({preferred}, storage.blob_path(blob_row.digest))
# Ensure there are no danglings OCI tags.
if check_oci_tags:
oci_tags = {t.id for t in Tag.select()}
referenced_oci_tags = {t.tag_id for t in TagToRepositoryTag.select()}
assert not oci_tags - referenced_oci_tags
# Ensure all tags have valid manifests.
for manifest in {t.manifest for t in Tag.select()}:
# Ensure that the manifest's blobs all exist.
found_blobs = {
b.blob.content_checksum
for b in ManifestBlob.select().where(
ManifestBlob.manifest == manifest)
}
parsed = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(manifest.manifest_bytes),
manifest.media_type.name)
assert set(parsed.local_blob_digests) == found_blobs
def hash_password(password, salt=None):
salt = salt or bcrypt.gensalt()
salt = Bytes.for_string_or_unicode(salt).as_encoded_str()
return bcrypt.hashpw(password.encode("utf-8"), salt)
def test_create_placeholder_blobs_on_first_pull(self, test_name,
proxy_manifest_response):
test_params = storage_test_cases[test_name]
# no blob placeholders are created for manifest lists - we don't have
# the sub-manifests at manifest list creation time, so there's no way
# to know which blobs the sub-manifest has.
if test_params["manifest_type"] in [
DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE,
OCI_IMAGE_INDEX_CONTENT_TYPE,
]:
pytest.skip(
"manifest list detected - skipping blob placeholder test")
repo = f"{self.orgname}/{test_params['image_name']}"
params = {
"repository": repo,
"manifest_ref": test_params["manifest_ref"],
}
proxy_mock = proxy_manifest_response(test_params["manifest_ref"],
test_params["manifest_json"],
test_params["manifest_type"])
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
headers = _get_auth_headers(self.sub, self.ctx, repo)
headers["Accept"] = ", ".join(
DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union(
DOCKER_SCHEMA1_CONTENT_TYPES))
conduct_call(
self.client,
test_params["view_name"],
url_for,
"GET",
params,
expected_code=200,
headers=headers,
)
parsed = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(test_params["manifest_json"]),
test_params["manifest_type"],
sparse_manifest_support=True,
)
manifest = Manifest.filter(Manifest.digest == parsed.digest).get()
mdict = parsed.manifest_dict
layers = mdict.get("layers", mdict.get("fsLayers"))
mblobs = ManifestBlob.filter(ManifestBlob.manifest == manifest)
expected_count = len(layers)
# schema 2 manifests have an extra config blob which we need to take into
# consideration in the total count
config_digest = ""
if parsed.schema_version == 2:
config_digest = parsed.config.digest
expected_count += 1
assert mblobs.count() == expected_count
for mblob in mblobs:
blob = None
layer = None
# don't assert if digest belongs to a config blob
if mblob.blob.content_checksum == config_digest:
continue
for layer in layers:
digest = layer.get("digest", layer.get("blobSum"))
if mblob.blob.content_checksum == digest:
blob = mblob.blob
layer = layer
break
assert blob is not None
assert blob.image_size == layer.get("size", None)
# the absence of an image storage placement for a blob indicates that it's
# a placeholder blob, not yet downloaded from the upstream registry.
placements = ImageStoragePlacement.filter(
ImageStoragePlacement.storage == blob)
assert placements.count() == 0
def pull(
self,
session,
namespace,
repo_name,
tag_names,
images,
credentials=None,
expected_failure=None,
options=None,
):
options = options or ProtocolOptions()
scopes = options.scopes or ["repository:%s:pull" % self.repo_name(namespace, repo_name)]
tag_names = [tag_names] if isinstance(tag_names, str) else tag_names
# Ping!
self.ping(session)
# Perform auth and retrieve a token.
token, _ = self.auth(
session,
credentials,
namespace,
repo_name,
scopes=scopes,
expected_failure=expected_failure,
)
if token is None and not options.attempt_pull_without_token:
return None
headers = {}
if token:
headers = {
"Authorization": "Bearer " + token,
}
if self.schema == "oci":
headers["Accept"] = ",".join(
options.accept_mimetypes
if options.accept_mimetypes is not None
else OCI_CONTENT_TYPES
)
elif self.schema == "schema2":
headers["Accept"] = ",".join(
options.accept_mimetypes
if options.accept_mimetypes is not None
else DOCKER_SCHEMA2_CONTENT_TYPES
)
manifests = {}
image_ids = {}
for tag_name in tag_names:
# Retrieve the manifest for the tag or digest.
response = self.conduct(
session,
"GET",
"/v2/%s/manifests/%s" % (self.repo_name(namespace, repo_name), tag_name),
expected_status=(200, expected_failure, V2ProtocolSteps.GET_MANIFEST),
headers=headers,
)
if response.status_code == 401:
assert "WWW-Authenticate" in response.headers
response.encoding = "utf-8"
if expected_failure is not None:
return None
# Ensure the manifest returned by us is valid.
ct = response.headers["Content-Type"]
if self.schema == "schema1":
assert ct in DOCKER_SCHEMA1_CONTENT_TYPES
if options.require_matching_manifest_type:
if self.schema == "schema1":
assert ct in DOCKER_SCHEMA1_CONTENT_TYPES
if self.schema == "schema2":
assert ct in DOCKER_SCHEMA2_CONTENT_TYPES
if self.schema == "oci":
assert ct in OCI_CONTENT_TYPES
manifest = parse_manifest_from_bytes(Bytes.for_string_or_unicode(response.text), ct)
manifests[tag_name] = manifest
if manifest.schema_version == 1:
image_ids[tag_name] = manifest.leaf_layer_v1_image_id
# Verify the blobs.
layer_index = 0
empty_count = 0
blob_digests = list(manifest.blob_digests)
for image in images:
if manifest.schema_version == 2 and image.is_empty:
empty_count += 1
continue
# If the layer is remote, then we expect the blob to *not* exist in the system.
blob_digest = blob_digests[layer_index]
expected_status = 404 if image.urls else 200
result = self.conduct(
session,
"GET",
"/v2/%s/blobs/%s" % (self.repo_name(namespace, repo_name), blob_digest),
expected_status=(expected_status, expected_failure, V2ProtocolSteps.GET_BLOB),
headers=headers,
options=options,
)
if expected_status == 200:
assert result.content == image.bytes
layer_index += 1
assert (len(blob_digests) + empty_count) >= len(
images
) # OCI/Schema 2 has 1 extra for config
return PullResult(manifests=manifests, image_ids=image_ids)
def pull_list(
self,
session,
namespace,
repo_name,
tag_names,
manifestlist,
credentials=None,
expected_failure=None,
options=None,
):
options = options or ProtocolOptions()
scopes = options.scopes or [
"repository:%s:push,pull" % self.repo_name(namespace, repo_name)
]
tag_names = [tag_names] if isinstance(tag_names, str) else tag_names
# Ping!
self.ping(session)
# Perform auth and retrieve a token.
token, _ = self.auth(
session,
credentials,
namespace,
repo_name,
scopes=scopes,
expected_failure=expected_failure,
)
if token is None:
assert V2Protocol.FAILURE_CODES[V2ProtocolSteps.AUTH].get(expected_failure)
return
headers = {
"Authorization": "Bearer " + token,
"Accept": ",".join(MANIFEST_LIST_TYPES),
}
for tag_name in tag_names:
# Retrieve the manifest for the tag or digest.
response = self.conduct(
session,
"GET",
"/v2/%s/manifests/%s" % (self.repo_name(namespace, repo_name), tag_name),
expected_status=(200, expected_failure, V2ProtocolSteps.GET_MANIFEST_LIST),
headers=headers,
)
if expected_failure is not None:
return None
# Parse the returned manifest list and ensure it matches.
ct = response.headers["Content-Type"]
assert is_manifest_list_type(ct), "Expected list type, found: %s" % ct
retrieved = parse_manifest_from_bytes(Bytes.for_string_or_unicode(response.text), ct)
assert retrieved.schema_version == 2
assert retrieved.is_manifest_list
assert retrieved.digest == manifestlist.digest
# Pull each of the manifests inside and ensure they can be retrieved.
for manifest_digest in retrieved.child_manifest_digests():
response = self.conduct(
session,
"GET",
"/v2/%s/manifests/%s" % (self.repo_name(namespace, repo_name), manifest_digest),
expected_status=(200, expected_failure, V2ProtocolSteps.GET_MANIFEST),
headers=headers,
)
if expected_failure is not None:
return None
ct = response.headers["Content-Type"]
manifest = parse_manifest_from_bytes(Bytes.for_string_or_unicode(response.text), ct)
assert not manifest.is_manifest_list
assert manifest.digest == manifest_digest
def _get_test_file_contents(test_name, kind):
filename = '%s.%s.json' % (test_name, kind)
data_dir = os.path.dirname(__file__)
with open(os.path.join(data_dir, 'conversion_data', filename), 'r') as f:
return Bytes.for_string_or_unicode(f.read())
def test_parse_manifest_from_bytes(media_type, manifest_bytes):
assert parse_manifest_from_bytes(
Bytes.for_string_or_unicode(manifest_bytes),
media_type,
validate=False)
def python_value(self, value):
if value is None:
return None
return Credential(Bytes.for_string_or_unicode(value).as_encoded_str())
def build_schema2(self, images, blobs, options):
builder = DockerSchema2ManifestBuilder()
for image in images:
checksum = "sha256:" + hashlib.sha256(image.bytes).hexdigest()
if image.urls is None:
blobs[checksum] = image.bytes
# If invalid blob references were requested, just make it up.
if options.manifest_invalid_blob_references:
checksum = "sha256:" + hashlib.sha256(
"notarealthing").hexdigest()
if not image.is_empty:
builder.add_layer(checksum, len(image.bytes), urls=image.urls)
def history_for_image(image):
history = {
"created":
"2018-04-03T18:37:09.284840891Z",
"created_by":
(("/bin/sh -c #(nop) ENTRYPOINT %s" %
image.config["Entrypoint"])
if image.config and image.config.get("Entrypoint") else
"/bin/sh -c #(nop) %s" % image.id),
}
if image.is_empty:
history["empty_layer"] = True
return history
config = {
"os": "linux",
"rootfs": {
"type": "layers",
"diff_ids": []
},
"history": [history_for_image(image) for image in images],
}
if options.with_broken_manifest_config:
# NOTE: We are missing the history entry on purpose.
config = {
"os": "linux",
"rootfs": {
"type": "layers",
"diff_ids": []
},
}
if images and images[-1].config:
config["config"] = images[-1].config
config_json = json.dumps(config, ensure_ascii=options.ensure_ascii)
schema2_config = DockerSchema2Config(
Bytes.for_string_or_unicode(config_json),
skip_validation_for_testing=options.with_broken_manifest_config,
)
builder.set_config(schema2_config)
blobs[schema2_config.digest] = schema2_config.bytes.as_encoded_str()
return builder.build(ensure_ascii=options.ensure_ascii)
def retarget_tag(
tag_name,
manifest_id,
is_reversion=False,
now_ms=None,
raise_on_error=False,
):
"""
Creates or updates a tag with the specified name to point to the given manifest under its
repository.
If this action is a reversion to a previous manifest, is_reversion should be set to True.
Returns the newly created tag row or None on error.
"""
try:
manifest = (Manifest.select(
Manifest,
MediaType).join(MediaType).where(Manifest.id == manifest_id).get())
except Manifest.DoesNotExist:
if raise_on_error:
raise RetargetTagException("Manifest requested no longer exists")
return None
# CHECK: Make sure that we are not mistargeting a schema 1 manifest to a tag with a different
# name.
if manifest.media_type.name in DOCKER_SCHEMA1_CONTENT_TYPES:
try:
parsed = DockerSchema1Manifest(Bytes.for_string_or_unicode(
manifest.manifest_bytes),
validate=False)
if parsed.tag != tag_name:
logger.error(
"Tried to re-target schema1 manifest with tag `%s` to tag `%s",
parsed.tag,
tag_name,
)
return None
except MalformedSchema1Manifest as msme:
logger.exception("Could not parse schema1 manifest")
if raise_on_error:
raise RetargetTagException(msme)
return None
legacy_image = get_legacy_image_for_manifest(manifest)
now_ms = now_ms or get_epoch_timestamp_ms()
now_ts = int(now_ms // 1000)
with db_transaction():
# Lookup an existing tag in the repository with the same name and, if present, mark it
# as expired.
existing_tag = get_tag(manifest.repository_id, tag_name)
if existing_tag is not None:
_, okay = set_tag_end_ms(existing_tag, now_ms)
# TODO: should we retry here and/or use a for-update?
if not okay:
return None
# Create a new tag pointing to the manifest with a lifetime start of now.
created = Tag.create(
name=tag_name,
repository=manifest.repository_id,
lifetime_start_ms=now_ms,
reversion=is_reversion,
manifest=manifest,
tag_kind=Tag.tag_kind.get_id("tag"),
)
return created
def test_config_missing_required():
valid_index = json.loads(SAMPLE_INDEX)
valid_index.pop("schemaVersion")
with pytest.raises(MalformedIndex):
OCIIndex(Bytes.for_string_or_unicode(json.dumps(valid_index)))
def test_malformed_manifest_lists(json_data):
with pytest.raises(MalformedSchema2ManifestList):
DockerSchema2ManifestList(Bytes.for_string_or_unicode(json_data))
def assert_gc_integrity(expect_storage_removed=True):
"""
Specialized assertion for ensuring that GC cleans up all dangling storages and labels, invokes
the callback for images removed and doesn't invoke the callback for images *not* removed.
"""
# Add a callback for when images are removed.
removed_image_storages = []
remove_callback = model.config.register_image_cleanup_callback(
removed_image_storages.extend)
# Store existing storages. We won't verify these for existence because they
# were likely created as test data.
existing_digests = set()
for storage_row in ImageStorage.select():
if storage_row.cas_path:
existing_digests.add(storage_row.content_checksum)
for blob_row in ApprBlob.select():
existing_digests.add(blob_row.digest)
# Store the number of dangling objects.
existing_storage_count = _get_dangling_storage_count()
existing_label_count = _get_dangling_label_count()
existing_manifest_count = _get_dangling_manifest_count()
# Yield to the GC test.
with check_transitive_modifications():
try:
yield
finally:
remove_callback()
# Ensure the number of dangling storages, manifests and labels has not changed.
updated_storage_count = _get_dangling_storage_count()
assert updated_storage_count == existing_storage_count
updated_label_count = _get_dangling_label_count()
assert updated_label_count == existing_label_count, _get_dangling_labels()
updated_manifest_count = _get_dangling_manifest_count()
assert updated_manifest_count == existing_manifest_count
# Ensure that for each call to the image+storage cleanup callback, the image and its
# storage is not found *anywhere* in the database.
for removed_image_and_storage in removed_image_storages:
assert isinstance(removed_image_and_storage, Image)
try:
# NOTE: SQLite can and will reuse AUTOINCREMENT IDs occasionally, so if we find a row
# with the same ID, make sure it does not have the same Docker Image ID.
# See: https://www.sqlite.org/autoinc.html
found_image = Image.get(id=removed_image_and_storage.id)
assert (found_image.docker_image_id !=
removed_image_and_storage.docker_image_id
), "Found unexpected removed image %s under repo %s" % (
found_image.id,
found_image.repository,
)
except Image.DoesNotExist:
pass
# Ensure that image storages are only removed if not shared.
shared = Image.select().where(
Image.storage == removed_image_and_storage.storage_id).count()
if shared == 0:
shared = (ManifestBlob.select().where(
ManifestBlob.blob ==
removed_image_and_storage.storage_id).count())
if shared == 0:
shared = (UploadedBlob.select().where(
UploadedBlob.blob ==
removed_image_and_storage.storage_id).count())
if shared == 0:
with pytest.raises(ImageStorage.DoesNotExist):
ImageStorage.get(id=removed_image_and_storage.storage_id)
with pytest.raises(ImageStorage.DoesNotExist):
ImageStorage.get(uuid=removed_image_and_storage.storage.uuid)
# Ensure all CAS storage is in the storage engine.
preferred = storage.preferred_locations[0]
for storage_row in ImageStorage.select():
if storage_row.content_checksum in existing_digests:
continue
if storage_row.cas_path:
storage.get_content({preferred},
storage.blob_path(
storage_row.content_checksum))
for blob_row in ApprBlob.select():
if blob_row.digest in existing_digests:
continue
storage.get_content({preferred}, storage.blob_path(blob_row.digest))
# Ensure all tags have valid manifests.
for manifest in {t.manifest for t in Tag.select()}:
# Ensure that the manifest's blobs all exist.
found_blobs = {
b.blob.content_checksum
for b in ManifestBlob.select().where(
ManifestBlob.manifest == manifest)
}
parsed = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(manifest.manifest_bytes),
manifest.media_type.name)
assert set(parsed.local_blob_digests) == found_blobs
def test_valid_remote_manifest():
manifest = DockerSchema2Manifest(
Bytes.for_string_or_unicode(REMOTE_MANIFEST_BYTES))
assert manifest.config.size == 1885
assert (
str(manifest.config.digest) ==
"sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7"
)
assert manifest.media_type == "application/vnd.docker.distribution.manifest.v2+json"
assert manifest.has_remote_layer
assert len(manifest.filesystem_layers) == 4
assert manifest.filesystem_layers[0].compressed_size == 1234
assert (
str(manifest.filesystem_layers[0].digest) ==
"sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736"
)
assert manifest.filesystem_layers[0].is_remote
assert manifest.filesystem_layers[0].urls == ["http://some/url"]
assert manifest.leaf_filesystem_layer == manifest.filesystem_layers[3]
assert not manifest.leaf_filesystem_layer.is_remote
assert manifest.leaf_filesystem_layer.compressed_size == 73109
expected = set([str(layer.digest)
for layer in manifest.filesystem_layers] +
[manifest.config.digest])
blob_digests = set(manifest.blob_digests)
local_digests = set(manifest.local_blob_digests)
assert blob_digests == expected
assert local_digests == (expected - {manifest.filesystem_layers[0].digest})
assert manifest.has_remote_layer
assert manifest.get_leaf_layer_v1_image_id(None) is None
assert manifest.get_legacy_image_ids(None) is None
retriever = ContentRetrieverForTesting.for_config(
{
"config": {
"Labels": {},
},
"rootfs": {
"type": "layers",
"diff_ids": []
},
"history": [
{
"created": "2018-04-03T18:37:09.284840891Z",
"created_by": "foo"
},
{
"created": "2018-04-12T18:37:09.284840891Z",
"created_by": "bar"
},
{
"created": "2018-04-03T18:37:09.284840891Z",
"created_by": "foo"
},
{
"created": "2018-04-12T18:37:09.284840891Z",
"created_by": "bar"
},
],
},
"sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7",
1885,
)
manifest_image_layers = list(manifest.get_layers(retriever))
assert len(manifest_image_layers) == len(list(manifest.filesystem_layers))
for index in range(0, 4):
assert manifest_image_layers[index].blob_digest == str(
manifest.filesystem_layers[index].digest)
def build(self, json_web_key=None, ensure_ascii=True):
"""
Builds a DockerSchema1Manifest object, with optional signature.
NOTE: For backward compatibility, "JWS JSON Serialization" is used instead of "JWS Compact Serialization", since the latter **requires** that the
"alg" headers be carried in the **protected** headers, which was never done before migrating to authlib (One shouldn't be using schema1 anyways)
References:
- https://tools.ietf.org/html/rfc7515#section-10.7
- https://docs.docker.com/registry/spec/manifest-v2-1/#signed-manifests
"""
payload = OrderedDict(self._base_payload)
payload.update({
DOCKER_SCHEMA1_HISTORY_KEY: self._history,
DOCKER_SCHEMA1_FS_LAYERS_KEY: self._fs_layer_digests,
})
payload_str = json.dumps(payload, indent=3, ensure_ascii=ensure_ascii)
if json_web_key is None:
return DockerSchema1Manifest(
Bytes.for_string_or_unicode(payload_str))
payload_str = Bytes.for_string_or_unicode(payload_str).as_encoded_str()
split_point = payload_str.rfind(b"\n}")
protected_payload = {
DOCKER_SCHEMA1_FORMAT_TAIL_KEY:
base64url_encode(payload_str[split_point:]).decode("ascii"),
DOCKER_SCHEMA1_FORMAT_LENGTH_KEY:
split_point,
"time":
datetime.utcnow().strftime(_ISO_DATETIME_FORMAT_ZULU),
}
# Flattened JSON serialization header
jws = JsonWebSignature(algorithms=[_JWS_SIGNING_ALGORITHM])
headers = {
"protected": protected_payload,
"header": {
"alg": _JWS_SIGNING_ALGORITHM
},
}
signed = jws.serialize_json(headers, payload_str,
json_web_key.get_private_key())
protected = signed["protected"]
signature = signed["signature"]
logger.debug("Generated signature: %s", signature)
logger.debug("Generated protected block: %s", protected)
public_members = set(json_web_key.REQUIRED_JSON_FIELDS +
json_web_key.ALLOWED_PARAMS)
public_key = {
comp: value
for comp, value in list(json_web_key.as_dict().items())
if comp in public_members
}
public_key["kty"] = json_web_key.kty
signature_block = {
DOCKER_SCHEMA1_HEADER_KEY: {
"jwk": public_key,
"alg": _JWS_SIGNING_ALGORITHM
},
DOCKER_SCHEMA1_SIGNATURE_KEY: signature,
DOCKER_SCHEMA1_PROTECTED_KEY: protected,
}
logger.debug("Encoded signature block: %s",
json.dumps(signature_block))
payload.update({DOCKER_SCHEMA1_SIGNATURES_KEY: [signature_block]})
json_str = json.dumps(payload, indent=3, ensure_ascii=ensure_ascii)
return DockerSchema1Manifest(Bytes.for_string_or_unicode(json_str))
def test_invalid_index():
with pytest.raises(MalformedIndex):
OCIIndex(Bytes.for_string_or_unicode("{}"))