def get_legacy_image(self,
repository_ref,
docker_image_id,
storage,
include_blob=False):
"""
Returns the matching LegacyImage under the matching repository, if any.
If none, returns None.
"""
retriever = RepositoryContentRetriever(repository_ref._db_id, storage)
# Resolves the manifest and the layer index from the synthetic ID.
manifest, layer_index = self._resolve_legacy_image_id(docker_image_id)
if manifest is None:
return None
# Lookup the legacy image for the index.
legacy_image = manifest.lookup_legacy_image(layer_index, retriever)
if legacy_image is None or not include_blob:
return legacy_image
# If a blob was requested, load it into the legacy image.
return legacy_image.with_blob(
self.get_repo_blob_by_digest(repository_ref,
legacy_image.blob_digest,
include_placements=True))
def get_schema1_parsed_manifest(
self, manifest, namespace_name, repo_name, tag_name, storage, raise_on_error=False
):
"""
Returns the schema 1 manifest for this manifest, or None if none.
"""
try:
parsed = manifest.get_parsed_manifest()
except ManifestException:
if raise_on_error:
raise ManifestException
return None
try:
manifest_row = database.Manifest.get(id=manifest._db_id)
except database.Manifest.DoesNotExist:
if raise_on_error:
raise ManifestDoesNotExist
return None
retriever = RepositoryContentRetriever(manifest_row.repository_id, storage)
schema1 = parsed.get_schema1_manifest(namespace_name, repo_name, tag_name, retriever)
if schema1 is None and raise_on_error:
raise ManifestException
return schema1
def _list_manifest_layers(self,
repo_id,
parsed,
storage,
include_placements=False,
by_manifest=False):
""" Returns an *ordered list* of the layers found in the manifest, starting at the base and
working towards the leaf, including the associated Blob and its placements (if specified).
Returns None if the manifest could not be parsed and validated.
"""
assert not parsed.is_manifest_list
retriever = RepositoryContentRetriever(repo_id, storage)
requires_empty_blob = parsed.get_requires_empty_layer_blob(retriever)
storage_map = {}
blob_digests = list(parsed.local_blob_digests)
if requires_empty_blob:
blob_digests.append(EMPTY_LAYER_BLOB_DIGEST)
if blob_digests:
blob_query = self._lookup_repo_storages_by_content_checksum(
repo_id, blob_digests, by_manifest=by_manifest)
storage_map = {blob.content_checksum: blob for blob in blob_query}
layers = parsed.get_layers(retriever)
if layers is None:
logger.error("Could not load layers for manifest `%s`",
parsed.digest)
return None
manifest_layers = []
for layer in layers:
if layer.is_remote:
manifest_layers.append(ManifestLayer(layer, None))
continue
digest_str = str(layer.blob_digest)
if digest_str not in storage_map:
logger.error("Missing digest `%s` for manifest `%s`",
layer.blob_digest, parsed.digest)
return None
image_storage = storage_map[digest_str]
assert image_storage.cas_path is not None
assert image_storage.image_size is not None
placements = None
if include_placements:
placements = list(
model.storage.get_storage_locations(image_storage.uuid))
blob = Blob.for_image_storage(
image_storage,
storage_path=model.storage.get_layer_path(image_storage),
placements=placements,
)
manifest_layers.append(ManifestLayer(layer, blob))
return manifest_layers
def get_schema1_parsed_manifest(self, manifest, namespace_name, repo_name, tag_name, storage):
""" Returns the schema 1 manifest for this manifest, or None if none. """
try:
parsed = manifest.get_parsed_manifest()
except ManifestException:
return None
try:
manifest_row = database.Manifest.get(id=manifest._db_id)
except database.Manifest.DoesNotExist:
return None
retriever = RepositoryContentRetriever(manifest_row.repository_id, storage)
return parsed.get_schema1_manifest(namespace_name, repo_name, tag_name, retriever)
def get_tag_legacy_image_id(self, repository_ref, tag_name, storage):
"""
Returns the legacy image ID for the tag in the repository. If there is no legacy image,
returns None.
"""
tag = self.get_repo_tag(repository_ref, tag_name)
if tag is None:
return None
retriever = RepositoryContentRetriever(repository_ref.id, storage)
legacy_image = tag.manifest.lookup_legacy_image(0, retriever)
if legacy_image is None:
return None
return legacy_image.docker_image_id
def convert_manifest(self, manifest, namespace_name, repo_name, tag_name,
allowed_mediatypes, storage):
try:
parsed = manifest.get_parsed_manifest()
except ManifestException:
return None
try:
manifest_row = database.Manifest.get(id=manifest._db_id)
except database.Manifest.DoesNotExist:
return None
retriever = RepositoryContentRetriever(manifest_row.repository_id,
storage)
return parsed.convert_manifest(allowed_mediatypes, namespace_name,
repo_name, tag_name, retriever)
def test_repository_tags(repo_namespace, repo_name, registry_model):
repository_ref = registry_model.lookup_repository(repo_namespace,
repo_name)
tags = registry_model.list_all_active_repository_tags(repository_ref)
assert len(tags)
tags_map = registry_model.get_legacy_tags_map(repository_ref, storage)
for tag in tags:
found_tag = registry_model.get_repo_tag(repository_ref, tag.name)
assert found_tag == tag
retriever = RepositoryContentRetriever(repository_ref.id, storage)
legacy_image = tag.manifest.lookup_legacy_image(0, retriever)
found_image = registry_model.get_legacy_image(
repository_ref, found_tag.manifest.legacy_image_root_id, storage)
if found_image is not None:
assert found_image.docker_image_id == legacy_image.docker_image_id
assert tags_map[tag.name] == found_image.docker_image_id
def test_retriever(initialized_db):
repository = create_repository("devtable", "newrepo", None)
layer_json = json.dumps({
"config": {},
"rootfs": {
"type": "layers",
"diff_ids": []
},
"history": [
{
"created": "2018-04-03T18:37:09.284840891Z",
"created_by": "do something",
},
{
"created": "2018-04-03T18:37:09.284840891Z",
"created_by": "do something",
},
],
})
# Add a blob containing the config.
_, config_digest = _populate_blob(layer_json)
# Add a blob of random data.
random_data = "hello world"
_, random_digest = _populate_blob(random_data)
# Add another blob of random data.
other_random_data = "hi place"
_, other_random_digest = _populate_blob(other_random_data)
remote_digest = sha256_digest(b"something")
builder = DockerSchema2ManifestBuilder()
builder.set_config_digest(config_digest, len(layer_json.encode("utf-8")))
builder.add_layer(other_random_digest,
len(other_random_data.encode("utf-8")))
builder.add_layer(random_digest, len(random_data.encode("utf-8")))
manifest = builder.build()
assert config_digest in manifest.blob_digests
assert random_digest in manifest.blob_digests
assert other_random_digest in manifest.blob_digests
assert config_digest in manifest.local_blob_digests
assert random_digest in manifest.local_blob_digests
assert other_random_digest in manifest.local_blob_digests
# Write the manifest.
created_tuple = get_or_create_manifest(repository, manifest, storage)
assert created_tuple is not None
created_manifest = created_tuple.manifest
assert created_manifest
assert created_manifest.media_type.name == manifest.media_type
assert created_manifest.digest == manifest.digest
# Verify the linked blobs.
blob_digests = {
mb.blob.content_checksum
for mb in ManifestBlob.select().where(
ManifestBlob.manifest == created_manifest)
}
assert random_digest in blob_digests
assert other_random_digest in blob_digests
assert config_digest in blob_digests
# Delete any Image rows linking to the blobs from temp tags.
for blob_digest in blob_digests:
storage_row = ImageStorage.get(content_checksum=blob_digest)
for image in list(Image.select().where(Image.storage == storage_row)):
all_temp = all([
rt.hidden for rt in RepositoryTag.select().where(
RepositoryTag.image == image)
])
if all_temp:
RepositoryTag.delete().where(
RepositoryTag.image == image).execute()
image.delete_instance(recursive=True)
# Verify the blobs in the retriever.
retriever = RepositoryContentRetriever(repository, storage)
assert (retriever.get_manifest_bytes_with_digest(
created_manifest.digest) == manifest.bytes.as_encoded_str())
for blob_digest in blob_digests:
assert retriever.get_blob_bytes_with_digest(blob_digest) is not None