def create_manifest_for_testing(repository, differentiation_field="1"):
# Populate a manifest.
layer_json = json.dumps({
"config": {},
"rootfs": {
"type": "layers",
"diff_ids": []
},
"history": [],
})
# Add a blob containing the config.
_, config_digest = _populate_blob(layer_json)
remote_digest = sha256_digest("something")
builder = DockerSchema2ManifestBuilder()
builder.set_config_digest(config_digest, len(layer_json.encode("utf-8")))
builder.add_layer(remote_digest,
1234,
urls=["http://hello/world" + differentiation_field])
manifest = builder.build()
created = get_or_create_manifest(repository, manifest, storage)
assert created
return created.manifest, manifest
def _populate_blob(repo, content):
digest = str(sha256_digest(content))
location = ImageStorageLocation.get(name="local_us")
storage.put_content(["local_us"], storage.blob_path(digest), "somecontent")
blob = model.blob.store_blob_record_and_temp_link_in_repo(
repo, digest, location, len(content), 120)
return blob, digest
def _populate_blob(content):
digest = str(sha256_digest(content))
location = ImageStorageLocation.get(name="local_us")
blob = store_blob_record_and_temp_link("devtable", "newrepo", digest,
location, len(content), 120)
storage.put_content(["local_us"], get_layer_path(blob), content)
return blob, digest
def create_manifest_for_testing(repository,
differentiation_field='1',
include_shared_blob=False):
# Populate a manifest.
layer_json = json.dumps({
'config': {},
"rootfs": {
"type": "layers",
"diff_ids": []
},
"history": [],
})
# Add a blob containing the config.
_, config_digest = _populate_blob(layer_json)
remote_digest = sha256_digest('something')
builder = DockerSchema2ManifestBuilder()
builder.set_config_digest(config_digest, len(layer_json))
builder.add_layer(remote_digest,
1234,
urls=['http://hello/world' + differentiation_field])
if include_shared_blob:
_, blob_digest = _populate_blob('some data here')
builder.add_layer(blob_digest, 4567)
manifest = builder.build()
created = get_or_create_manifest(repository, manifest, storage)
assert created
return created.manifest, manifest
def _populate_blob(repo, content):
assert isinstance(repo, Repository)
assert isinstance(content, bytes)
digest = str(sha256_digest(content))
location = ImageStorageLocation.get(name="local_us")
blob = model.blob.store_blob_record_and_temp_link_in_repo(
repo, digest, location, len(content), TEMP_BLOB_EXPIRATION)
return blob, digest
def _populate_blob(repo, content):
assert isinstance(content, bytes)
digest = sha256_digest(content)
location = ImageStorageLocation.get(name="local_us")
storage.put_content(["local_us"], storage.blob_path(digest), content)
blob = model.blob.store_blob_record_and_temp_link_in_repo(
repo, digest, location, len(content), 120)
return blob, digest
def get_blob(layer):
content = Bytes.for_string_or_unicode(layer).as_encoded_str()
digest = str(sha256_digest(content))
blob = store_blob_record_and_temp_link(
self.orgname,
self.upstream_repository,
digest,
ImageStorageLocation.get(name="local_us"),
len(content),
120,
)
storage.put_content(["local_us"], get_layer_path(blob), content)
return blob, digest
def test_create_manifest_cannot_load_config_blob(initialized_db):
repository = create_repository("devtable", "newrepo", None)
layer_json = json.dumps({
"config": {},
"rootfs": {
"type": "layers",
"diff_ids": []
},
"history": [
{
"created": "2018-04-03T18:37:09.284840891Z",
"created_by": "do something",
},
],
})
# Add a blob containing the config.
_, config_digest = _populate_blob(layer_json)
# Add a blob of random data.
random_data = "hello world"
_, random_digest = _populate_blob(random_data)
remote_digest = sha256_digest(b"something")
builder = DockerSchema2ManifestBuilder()
builder.set_config_digest(config_digest, len(layer_json.encode("utf-8")))
builder.add_layer(random_digest, len(random_data.encode("utf-8")))
manifest = builder.build()
broken_retriever = BrokenRetriever()
# Write the manifest.
with pytest.raises(CreateManifestException):
get_or_create_manifest(repository,
manifest,
storage,
retriever=broken_retriever,
raise_on_error=True)
def digest(self):
return digest_tools.sha256_digest(self._payload.as_encoded_str())
def digest(self):
"""
Returns the digest of this config object.
"""
return digest_tools.sha256_digest(self._config_bytes.as_encoded_str())
def digest(self):
return digest_tools.sha256_digest(self._payload)
def digest(self):
"""
The digest of the manifest, including type prefix.
"""
return digest_tools.sha256_digest(self._manifest_bytes.as_encoded_str())
def test_retriever(initialized_db):
repository = create_repository("devtable", "newrepo", None)
layer_json = json.dumps({
"config": {},
"rootfs": {
"type": "layers",
"diff_ids": []
},
"history": [
{
"created": "2018-04-03T18:37:09.284840891Z",
"created_by": "do something",
},
{
"created": "2018-04-03T18:37:09.284840891Z",
"created_by": "do something",
},
],
})
# Add a blob containing the config.
_, config_digest = _populate_blob(layer_json)
# Add a blob of random data.
random_data = "hello world"
_, random_digest = _populate_blob(random_data)
# Add another blob of random data.
other_random_data = "hi place"
_, other_random_digest = _populate_blob(other_random_data)
remote_digest = sha256_digest(b"something")
builder = DockerSchema2ManifestBuilder()
builder.set_config_digest(config_digest, len(layer_json.encode("utf-8")))
builder.add_layer(other_random_digest,
len(other_random_data.encode("utf-8")))
builder.add_layer(random_digest, len(random_data.encode("utf-8")))
manifest = builder.build()
assert config_digest in manifest.blob_digests
assert random_digest in manifest.blob_digests
assert other_random_digest in manifest.blob_digests
assert config_digest in manifest.local_blob_digests
assert random_digest in manifest.local_blob_digests
assert other_random_digest in manifest.local_blob_digests
# Write the manifest.
created_tuple = get_or_create_manifest(repository, manifest, storage)
assert created_tuple is not None
created_manifest = created_tuple.manifest
assert created_manifest
assert created_manifest.media_type.name == manifest.media_type
assert created_manifest.digest == manifest.digest
# Verify the linked blobs.
blob_digests = {
mb.blob.content_checksum
for mb in ManifestBlob.select().where(
ManifestBlob.manifest == created_manifest)
}
assert random_digest in blob_digests
assert other_random_digest in blob_digests
assert config_digest in blob_digests
# Delete any Image rows linking to the blobs from temp tags.
for blob_digest in blob_digests:
storage_row = ImageStorage.get(content_checksum=blob_digest)
for image in list(Image.select().where(Image.storage == storage_row)):
all_temp = all([
rt.hidden for rt in RepositoryTag.select().where(
RepositoryTag.image == image)
])
if all_temp:
RepositoryTag.delete().where(
RepositoryTag.image == image).execute()
image.delete_instance(recursive=True)
# Verify the blobs in the retriever.
retriever = RepositoryContentRetriever(repository, storage)
assert (retriever.get_manifest_bytes_with_digest(
created_manifest.digest) == manifest.bytes.as_encoded_str())
for blob_digest in blob_digests:
assert retriever.get_blob_bytes_with_digest(blob_digest) is not None
def test_get_or_create_manifest_with_remote_layers(initialized_db):
repository = create_repository("devtable", "newrepo", None)
layer_json = json.dumps({
"config": {},
"rootfs": {
"type": "layers",
"diff_ids": []
},
"history": [
{
"created": "2018-04-03T18:37:09.284840891Z",
"created_by": "do something",
},
{
"created": "2018-04-03T18:37:09.284840891Z",
"created_by": "do something",
},
],
})
# Add a blob containing the config.
_, config_digest = _populate_blob(layer_json)
# Add a blob of random data.
random_data = "hello world"
_, random_digest = _populate_blob(random_data)
remote_digest = sha256_digest(b"something")
builder = DockerSchema2ManifestBuilder()
builder.set_config_digest(config_digest, len(layer_json.encode("utf-8")))
builder.add_layer(remote_digest, 1234, urls=["http://hello/world"])
builder.add_layer(random_digest, len(random_data.encode("utf-8")))
manifest = builder.build()
assert remote_digest in manifest.blob_digests
assert remote_digest not in manifest.local_blob_digests
assert manifest.has_remote_layer
assert not manifest.has_legacy_image
assert manifest.get_schema1_manifest("foo", "bar", "baz", None) is None
# Write the manifest.
created_tuple = get_or_create_manifest(repository, manifest, storage)
assert created_tuple is not None
created_manifest = created_tuple.manifest
assert created_manifest
assert created_manifest.media_type.name == manifest.media_type
assert created_manifest.digest == manifest.digest
assert created_manifest.config_media_type == manifest.config_media_type
assert created_manifest.layers_compressed_size == manifest.layers_compressed_size
# Verify the legacy image.
legacy_image = get_legacy_image_for_manifest(created_manifest)
assert legacy_image is None
# Verify the linked blobs.
blob_digests = {
mb.blob.content_checksum
for mb in ManifestBlob.select().where(
ManifestBlob.manifest == created_manifest)
}
assert random_digest in blob_digests
assert config_digest in blob_digests
assert remote_digest not in blob_digests
