def get_test_data_local() -> Generator[TypingGetTestDataLocal, None, None]:
"""Dynamically initializes test data for a local mutable registry."""
images = [
{
"image": "{0}/library/python",
"tag": "3.7.2-slim-stretch",
"digests": {
DockerMediaTypes.DISTRIBUTION_MANIFEST_V2:
FormattedSHA256(
"0005ba40bf87e486d7061ca0112123270e4a6088b5071223c8d467db3dbba908"
),
},
"original_endpoint": Indices.DOCKERHUB,
"protocol": "http",
},
{
"image": "{0}/library/busybox",
"tag": "1.30.1",
"digests": {
DockerMediaTypes.DISTRIBUTION_MANIFEST_V2:
FormattedSHA256(
"4fe8827f51a5e11bb83afa8227cbccb402df840d32c6b633b7ad079bc8144100"
),
},
"original_endpoint": Indices.DOCKERHUB,
"protocol": "http",
},
]
for image in images:
yield image
def test___new__():
"""Test that a formatted SHA256 can be instantiated."""
digest = "0123456789012345678901234567890123456789012345678901234567890123"
formattedsha256 = FormattedSHA256(digest)
assert formattedsha256
assert formattedsha256.sha256 == digest # pylint: disable=no-member
assert str(formattedsha256) == f"sha256:{digest}"
digest = "sha256:0123456789012345678901234567890123456789012345678901234567890123"
formattedsha256 = FormattedSHA256(digest)
assert formattedsha256
assert formattedsha256.sha256 == digest[7:] # pylint: disable=no-member
assert str(formattedsha256) == digest
with pytest.raises(ValueError) as exc_info:
FormattedSHA256(None)
assert "None" in str(exc_info.value)
digest = "012345678901234567890123456789012345678901234567890123456789012"
with pytest.raises(ValueError) as exc_info:
FormattedSHA256(digest)
assert digest in str(exc_info.value)
digest = "sha1:0123456789012345678901234567890123456789012345678901234567890123"
with pytest.raises(ValueError) as exc_info:
FormattedSHA256(digest)
assert digest in str(exc_info.value)
async def test_verify_signatures_manipulated_signatures(
gnupg_keypair: GnuPGKeypair, image_config: ImageConfig
):
"""Test that signature verification detects manipulated signatures."""
signer = GPGSigner(
keyid=gnupg_keypair.keyid,
passphrase=gnupg_keypair.passphrase,
homedir=gnupg_keypair.gnupg_home,
)
# Add a single signature ...
await image_config.sign(signer)
response = await image_config.verify_signatures(
signer_kwargs={GPGSigner.__name__: {"homedir": gnupg_keypair.gnupg_home}}
)
assert response.results[0].valid
# Modify the digest value of the (first) signature ...
signatures = image_config.get_signature_list()
temp = deepcopy(signatures)
temp[0] = ImageConfigSignatureEntry(
digest=FormattedSHA256.calculate(b"tampertampertamper"),
signature=temp[0].signature,
)
image_config.set_signature_list(temp)
# An exception should be raised if digest value from the signature does not match the canonical digest of the image
# configuration (without any signatures).
with pytest.raises(DigestMismatchError) as exception:
await image_config.verify_signatures()
assert str(exception.value).startswith("Image config canonical digest mismatch:")
# Restore the unmodified signature and endorse ...
image_config.set_signature_list(signatures)
await image_config.sign(signer, SignatureTypes.ENDORSE)
response = await image_config.verify_signatures(
signer_kwargs={GPGSigner.__name__: {"homedir": gnupg_keypair.gnupg_home}}
)
assert response.results[0].valid
# Modify the digest value of the second signature ...
signatures = image_config.get_signature_list()
temp = deepcopy(signatures)
temp[1] = ImageConfigSignatureEntry(
digest=FormattedSHA256.calculate(b"tampertampertamper"),
signature=temp[1].signature,
)
image_config.set_signature_list(temp)
# An exception should be raised if digest value from the signature does not match the canonical digest of the image
# configuration (including the first signature).
with pytest.raises(DigestMismatchError) as exception:
await image_config.verify_signatures()
assert str(exception.value).startswith("Image config canonical digest mismatch:")
def test_set_tag(archive_repositories: ArchiveRepositories,
image_name: ImageName, name: str):
"""Test repository tag assignment."""
tag = archive_repositories.get_tag(image_name)
assert tag
assert FormattedSHA256(tag) == image_name.digest
digest = FormattedSHA256.calculate(name.encode("utf-8"))
name = ImageName.parse(name)
archive_repositories.set_tag(name, digest)
assert FormattedSHA256(archive_repositories.get_tag(name)) == digest
def test_get_config_digest(registry_v2_manifest: RegistryV2Manifest):
"""Test image configuration digest retrieval."""
formattedsha256 = FormattedSHA256(
"8f1196ff19e7b5c5861de192ae77e8d7a692fcbca2dd3174d324980f72ab49bf")
assert registry_v2_manifest.get_config_digest() == formattedsha256
assert (registry_v2_manifest.get_config_digest(
ImageName.parse("ignored")) == formattedsha256)
def get_manifests(self,
*,
architecture: str = None,
os: str = None) -> List[FormattedSHA256]:
"""
Retrieves the listing of manifest layer identifiers.
Args:
architecture: The name of the CPU architecture.
os: The name of the operating system.
Returns:
list: Manifest identifiers in the form: <hash type>:<digest value>.
"""
if self.get_media_type() not in [
DockerMediaTypes.DISTRIBUTION_MANIFEST_LIST_V2,
OCIMediaTypes.IMAGE_INDEX_V1,
]:
raise NotImplementedError(
f"Unsupported media type: {self.get_media_type()}")
result = []
for manifest in self.get_json()["manifests"]:
if (architecture and manifest.get("platform", "").get(
"architecture", "") != architecture):
continue
if os and manifest.get("platform", "").get("os", "") != os:
continue
result.append(FormattedSHA256.parse(manifest["digest"]))
return result
async def chunk_file(
file_in, file_out, *, file_in_is_async: bool = True, file_out_is_async: bool = True
) -> UtilChunkFile:
"""
Copies chunkcs from one file to another.
Args:
file_in: The file from which to retrieve the file chunks.
file_out: The file to which to store the file chunks.
file_in_is_async: If True, all file_in IO operations will be awaited.
file_out_is_async: If True, all file_out IO operations will be awaited.
Returns:
dict:
digest: The digest value of the chunked data.
size: The byte size of the chunked data in bytes.
"""
coroutine_read = file_in.read if file_in_is_async else async_wrap(file_in.read)
coroutine_write = (
file_out.write if file_out_is_async else async_wrap(file_out.write)
)
hasher = hashlib.sha256()
size = 0
while True:
chunk = await coroutine_read(CHUNK_SIZE)
if not chunk:
break
await coroutine_write(chunk)
hasher.update(chunk)
size += len(chunk)
await be_kind_rewind(file_out, file_is_async=file_out_is_async)
return {"digest": FormattedSHA256(hasher.hexdigest()), "size": size}
async def test_chunk_file(client_session: ClientSession, tmp_path: Path):
"""Tests that remote files can be chunked to disk."""
url = "https://tools.ietf.org/rfc/rfc2616.txt" # Hat tip
digest_expected = FormattedSHA256.parse(
"sha256:10211d2885196b97b1c78e1672f3f68ae97c294596ef2b7fd890cbd30a3427bf"
)
size = 422279
async with client_session.get(url=url,
allow_redirects=True) as client_response:
path_async = tmp_path.joinpath("test_async")
async with aiofiles.open(path_async, mode="w+b") as file:
result = await chunk_to_file(client_response, file)
assert result["client_response"] == client_response
assert result["digest"] == digest_expected
assert result["size"] == size
digest_actual = await hash_file(path_async)
assert digest_actual == digest_expected
path_sync = tmp_path.joinpath("test_sync")
async with client_session.get(url=url,
allow_redirects=True) as client_response:
with path_sync.open("w+b") as file:
result = await chunk_to_file(client_response,
file,
file_is_async=False)
assert result["client_response"] == client_response
assert result["digest"] == digest_expected
assert result["size"] == size
digest_actual = await hash_file(path_async)
assert digest_actual == digest_expected
async def put_image_layer(
self, image_name: ImageName, content,
**kwargs) -> DockerRegistryClientAsyncPutBlobUpload:
response = await self.docker_registry_client_async.post_blob(
image_name, **kwargs)
digest = FormattedSHA256.calculate(content)
return await self.docker_registry_client_async.put_blob_upload(
response["location"], digest, data=content, **kwargs)
def test_get_tag(archive_repositories: ArchiveRepositories,
image_name: ImageName):
"""Test repository tag retrieval."""
tag = archive_repositories.get_tag(image_name)
assert tag
assert FormattedSHA256(tag) == image_name.digest
assert not archive_repositories.get_tag(ImageName("does_not_exist"))
assert not archive_repositories.get_tag(ImageName("does_not", tag="exist"))
def get_digest_canonical(self) -> FormattedSHA256:
"""
Retrieves the SHA256 digest value of the image configuration in canonical JSON form.
Returns:
The SHA256 digest value of the image configuration in canonical JSON form.
"""
return FormattedSHA256.calculate(self.get_bytes_canonical())
async def put_image_layer_from_disk(self, image_name: ImageName, file,
**kwargs) -> FormattedSHA256:
# TODO: Convert to async
if self.dry_run:
LOGGER.debug("Dry Run: skipping put_image_layer_from_disk")
return FormattedSHA256("0" * 64)
# TODO: Do we really want to use random garbage here???
# Look into moby/.../save.go to find what to use instead.
digest = FormattedSHA256.calculate("{0}{1}{2}".format(
str(image_name), datetime.datetime.now(),
random.randint(1, 101)).encode("utf-8"))
layer = ArchiveManifest.digest_to_layer(digest)
with open(self.archive, "rb+") as file_out:
tar_mkdir(file_out, os.path.dirname(layer))
file_out.seek(0)
tar(file_out, layer, file)
return digest
def get_config_digest(self,
image_name: ImageName = None) -> FormattedSHA256:
if self.get_media_type() not in [
DockerMediaTypes.DISTRIBUTION_MANIFEST_V2,
OCIMediaTypes.IMAGE_MANIFEST_V1,
]:
raise NotImplementedError("Unsupported media type: {0}".format(
self.get_media_type()))
return FormattedSHA256.parse(self.get_json()["config"]["digest"])
async def put_image_layer(self, image_name: ImageName, content,
**kwargs) -> FormattedSHA256:
# TODO: Do we really want to use random garbage here???
# Look into moby/.../save.go to find what to use instead.
digest = FormattedSHA256.calculate("{0}{1}{2}".format(
str(image_name), datetime.datetime.now(),
random.randint(1, 101)).encode("utf-8"))
layer = ArchiveManifest.digest_to_layer(digest)
with open(self.archive, "rb+") as file_out:
tar_add_file(file_out, layer, content)
return digest
def layer_to_digest(layer: str) -> FormattedSHA256:
"""
Coverts a archive layer identifier to a digest value.
Args:
layer: The archive layer identifier (relative tar path).
Returns:
The corresponding digest value in the form: <hash type>:<digest value>.
"""
return FormattedSHA256(layer[:-10])
def get_layers(self,
image_name: ImageName = None) -> List[FormattedSHA256]:
if self.get_media_type() not in [
DockerMediaTypes.DISTRIBUTION_MANIFEST_V2,
OCIMediaTypes.IMAGE_MANIFEST_V1,
]:
raise NotImplementedError("Unsupported media type: {0}".format(
self.get_media_type()))
return [
FormattedSHA256.parse(layer["digest"])
for layer in self.get_json()["layers"]
]
async def test_layer_exists(
registry_v2_image_source: RegistryV2ImageSource,
known_good_image: TypingKnownGoodImage,
**kwargs,
):
"""Test layer existence."""
LOGGER.debug("Retrieving manifest for: %s ...",
known_good_image["image_name"])
manifest = await registry_v2_image_source.get_manifest(
known_good_image["image_name"], **kwargs)
layer = manifest.get_layers()[-1]
assert await registry_v2_image_source.layer_exists(
known_good_image["image_name"], layer, **kwargs)
assert not await registry_v2_image_source.layer_exists(
known_good_image["image_name"], FormattedSHA256("0" * 64), **kwargs)
async def test_layer_exists(
registry_v2_image_source: RegistryV2ImageSource,
known_good_image_local: Dict,
**kwargs,
):
"""Test layer existence."""
if "protocol" in known_good_image_local:
kwargs["protocol"] = known_good_image_local["protocol"]
LOGGER.debug("Retrieving manifest for: %s ...",
known_good_image_local["image_name"])
manifest = await registry_v2_image_source.get_manifest(
known_good_image_local["image_name"], **kwargs)
layer = manifest.get_layers()[-1]
assert await registry_v2_image_source.layer_exists(
known_good_image_local["image_name"], layer, **kwargs)
assert not await registry_v2_image_source.layer_exists(
known_good_image_local["image_name"], FormattedSHA256("0" * 64), **
kwargs)
def get_combined_layerid(parent: FormattedSHA256,
layer: FormattedSHA256) -> FormattedSHA256:
"""
Retrieves the layer identifier for a given parent-layer combination.
Args:
parent: The parent layer identifier.
layer: The layer identifier.
Returns:
The corresponding layer identifier.
"""
result = layer
if parent:
# Note: The image layer is the digest value of the formatted string.
result = FormattedSHA256.calculate("{0} {1}".format(
parent, layer).encode("utf-8"))
return result
async def put_image_layer_from_disk(self, image_name: ImageName, file,
**kwargs) -> FormattedSHA256:
# pylint: disable=protected-access
file_is_async = kwargs.get("file_is_async", True)
if file_is_async:
file = (
file._file
) # DUCK PUNCH: Unwrap the file handle from the asynchronous object
# TODO: Do we really want to use random garbage here???
# Look into moby/.../save.go to find what to use instead.
digest = FormattedSHA256.calculate("{0}{1}{2}".format(
str(image_name), datetime.datetime.now(),
random.randint(1, 101)).encode("utf-8"))
layer = ArchiveManifest.digest_to_layer(digest)
with open(self.archive, "rb+") as file_out:
tar_mkdir(file_out, str(Path(layer).parent))
file_out.seek(0)
tar_add_file_from_disk(file_out, layer, file)
return digest
def append_manifest(self, manifest: "ArchiveManifest"):
"""
Appends an archive manifest to the archive changeset.
Args:
manifest: The archive manifest to be appended.
"""
# Remove the image if it already exists
self.remove_manifest(
FormattedSHA256(manifest.get_json()["Config"][:-5]))
# Remove all tags being assigned to the new image ...
tags = ArchiveChangeset.normalize_tags(manifest.get_tags())
if tags:
self.remove_tags(tags)
# Append the new image configuration ...
_json = self.get_json()
_json.append(manifest.get_json())
self._set_json(_json)
def test_remove_tags(archive_changeset: ArchiveChangeset):
"""Test repository tag removal"""
image_name = ImageName(
"base",
digest=FormattedSHA256(
"adecf4209bb9dd67d96393774cbd7f8bd2bad3596da42cde33daa0c41b14ac62"
),
tag="7.2",
)
tag = ArchiveChangeset.get_repository_tag(image_name)
manifest = archive_changeset.get_manifest(image_name)
assert tag in manifest.get_tags()
archive_changeset.remove_tags(["dummy"])
manifest = archive_changeset.get_manifest(image_name)
assert tag in manifest.get_tags()
archive_changeset.remove_tags([tag])
manifest = archive_changeset.get_manifest(image_name)
assert not manifest.get_tags()
def get_image_layers(self) -> List[FormattedSHA256]:
"""
Retrieves the listing of image layer identifiers.
Returns:
The listing of image layer identifiers.
"""
# Note: We need to handle both key cases, as Microsoft does not conform to the standard.
try:
rootfs = self.get_json()["rootfs"]
except KeyError:
rootfs = self.get_json()["rootfS"]
if rootfs is None:
raise MalformedConfigurationError(
"Unable to locate rootf[Ss] key within image configuration!",
config=self,
)
diff_ids = rootfs["diff_ids"]
return [FormattedSHA256.parse(x) for x in diff_ids]
def get_test_data() -> Generator[TypingGetTestData, None, None]:
"""Dynamically initializes test data."""
for endpoint in ["endpoint.io", "endpoint:port", None]:
for image in [
"image", "ns0/image", "ns0/ns1/image", "ns0/ns1/ns2/image"
]:
for tag in ["tag", None]:
for digest in [FormattedSHA256.calculate(b""), None]:
# Construct a complex string ...
string = image
if tag:
string = f"{string}:{tag}"
if digest:
string = f"{string}@{digest}"
if endpoint:
string = f"{endpoint}/{string}"
yield {
"digest": digest,
"endpoint": endpoint,
"image": image,
"object": ImageName.parse(string),
"string": string,
"tag": tag,
}
def get_config_digest(self,
image_name: ImageName = None) -> FormattedSHA256:
key = DeviceMapperRepositoryManifest._get_repository_key(image_name)
image = self.get_json()["Repositories"].get(key, {})
return FormattedSHA256.parse(image.get(str(image_name), None))
def formattedsha256() -> FormattedSHA256:
"""Provides a FormattedSHA256 instance with a distinct digest value."""
return FormattedSHA256(
"ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789")
def get_config_digest(self,
image_name: ImageName = None) -> FormattedSHA256:
config = self.get_config(image_name)
return FormattedSHA256(config["Config"][:-5])
def test_calculate():
"""Test that a formatted SHA256 can be calculated."""
assert (
FormattedSHA256.calculate(b"test data") ==
"sha256:916f0027a575074ce72a331777c3478d6513f786a591bd892da1a577bf2335f9"
)
def config_digest_signed_canonical(request) -> FormattedSHA256:
"""Provides the digest value of the canonical form of the signed sample image configuration."""
return FormattedSHA256.parse(
get_test_data(request, __name__, "config_signed_canonical.json.digest", "r")
)
def config_digest(request) -> FormattedSHA256:
"""Provides the digest value of the sample image configuration."""
return FormattedSHA256.parse(
get_test_data(request, __name__, "config.json.digest", "r")
)