def connect_manifests(manifests: list[Manifest], parent: Manifest,
repository_id: int):
"""
Connects manifests to a manifest list.
Raises a _ManifestAlreadyExists if any of the manifest children already exist.
"""
children = [
dict(manifest=parent,
child_manifest=manifest,
repository=repository_id) for manifest in manifests
]
try:
ManifestChild.insert_many(children).execute()
except IntegrityError as e:
raise _ManifestAlreadyExists(e)
def test_connect_existing_manifest_to_manifest_list(self, create_repo):
repo_ref = create_repo(self.orgname, self.upstream_repository,
self.user)
input_manifest = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(UBI8_LATEST_MANIFEST_SCHEMA2),
DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE,
)
proxy_model = ProxyModel(
self.orgname,
self.upstream_repository,
self.user,
)
manifest, _ = proxy_model._create_manifest_with_temp_tag(
repo_ref, input_manifest)
assert manifest is not None
input_list = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(UBI8_LATEST_MANIFEST_LIST),
DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE,
sparse_manifest_support=True,
)
manifest_list, _ = proxy_model._create_manifest_and_retarget_tag(
repo_ref, input_list, self.tag)
assert manifest_list is not None
conn_count = (ManifestChild.select().where(
ManifestChild.manifest == manifest_list.id,
ManifestChild.child_manifest == manifest.id,
).count())
assert conn_count == 1
def test_create_temp_tags_for_newly_created_sub_manifests_on_manifest_list(
self, create_repo):
repo_ref = create_repo(self.orgname, self.upstream_repository,
self.user)
input_manifest = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(UBI8_LATEST_MANIFEST_LIST),
DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE,
sparse_manifest_support=True,
)
proxy_model = ProxyModel(
self.orgname,
self.upstream_repository,
self.user,
)
manifest, _ = proxy_model._create_manifest_and_retarget_tag(
repo_ref, input_manifest, self.tag)
mchildren = ManifestChild.select(
ManifestChild.child_manifest_id).where(
ManifestChild.manifest == manifest.id)
tags = Tag.select().join(
ManifestChild,
on=(Tag.manifest_id == ManifestChild.child_manifest_id))
assert mchildren.count() == tags.count()
assert all([t.hidden
for t in tags]), "all sub manifest tags must be hidden"
assert all(
[t.name != self.tag for t in tags]
), "sub manifest tags must have temp tag name, not parent manifest name"
def _check_manifest_used(manifest_id):
assert manifest_id is not None
with db_transaction():
# Check if the manifest is referenced by any other tag.
try:
Tag.select().where(Tag.manifest == manifest_id).get()
return True
except Tag.DoesNotExist:
pass
# Check if the manifest is referenced as a child of another manifest.
try:
ManifestChild.select().where(ManifestChild.child_manifest == manifest_id).get()
return True
except ManifestChild.DoesNotExist:
pass
return False
def test_renew_manifest_and_parent_tag_when_manifest_is_child_of_manifest_list(
self, create_repo, proxy_manifest_response):
repo_ref = create_repo(self.orgname, self.upstream_repository,
self.user)
input_list = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(UBI8_LATEST_MANIFEST_LIST),
DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE,
sparse_manifest_support=True,
)
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock()):
proxy_model = ProxyModel(
self.orgname,
self.upstream_repository,
self.user,
)
manifest_list, tag = proxy_model._create_manifest_and_retarget_tag(
repo_ref, input_list, "latest")
assert manifest_list is not None
child = (ManifestChild.select(ManifestChild.child_manifest_id).join(
Manifest,
on=(ManifestChild.child_manifest_id == Manifest.id
)).where((ManifestChild.manifest_id == manifest_list.id)
& (Manifest.digest == UBI8_LATEST_DIGEST)))
manifest_tag = Tag.select().where(Tag.manifest == child).get()
manifest_list_tag = tag
proxy_mock = proxy_manifest_response(
UBI8_LATEST_DIGEST, UBI8_LATEST_MANIFEST_SCHEMA2,
DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE)
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
proxy_model = ProxyModel(
self.orgname,
self.upstream_repository,
self.user,
)
manifest = proxy_model.lookup_manifest_by_digest(
repo_ref, UBI8_LATEST_DIGEST)
updated_tag = oci.tag.get_tag_by_manifest_id(repo_ref.id, manifest.id)
updated_list_tag = oci.tag.get_tag_by_manifest_id(
repo_ref.id, manifest_list.id)
assert updated_tag.id == manifest_tag.id
assert updated_list_tag.id == manifest_list_tag.id
assert updated_tag.lifetime_end_ms > manifest_tag.lifetime_end_ms
assert updated_list_tag.lifetime_end_ms > manifest_list_tag.lifetime_end_ms
def test_create_sub_manifests_for_manifest_list(self, create_repo):
repo_ref = create_repo(self.orgname, self.upstream_repository,
self.user)
input_manifest = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(UBI8_LATEST_MANIFEST_LIST),
DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE,
sparse_manifest_support=True,
)
proxy_model = ProxyModel(
self.orgname,
self.upstream_repository,
self.user,
)
manifest, _ = proxy_model._create_manifest_and_retarget_tag(
repo_ref, input_manifest, self.tag)
mchildren = ManifestChild.select().where(
ManifestChild.manifest == manifest.id)
created_count = mchildren.count()
expected_count = len(
list(input_manifest.child_manifests(content_retriever=None)))
assert expected_count == created_count
def _garbage_collect_manifest(manifest_id, context):
assert manifest_id is not None
# Make sure the manifest isn't referenced.
if _check_manifest_used(manifest_id):
return False
# Add the manifest's blobs to the context to be GCed.
for manifest_blob in ManifestBlob.select().where(ManifestBlob.manifest == manifest_id):
context.add_blob_id(manifest_blob.blob_id)
# Retrieve the manifest's associated image, if any.
try:
legacy_image_id = ManifestLegacyImage.get(manifest=manifest_id).image_id
context.add_legacy_image_id(legacy_image_id)
except ManifestLegacyImage.DoesNotExist:
legacy_image_id = None
# Add child manifests to be GCed.
for connector in ManifestChild.select().where(ManifestChild.manifest == manifest_id):
context.add_manifest_id(connector.child_manifest_id)
# Add the labels to be GCed.
for manifest_label in ManifestLabel.select().where(ManifestLabel.manifest == manifest_id):
context.add_label_id(manifest_label.label_id)
# Delete the manifest.
with db_transaction():
try:
manifest = Manifest.select().where(Manifest.id == manifest_id).get()
except Manifest.DoesNotExist:
return False
assert manifest.id == manifest_id
assert manifest.repository_id == context.repository.id
if _check_manifest_used(manifest_id):
return False
# Delete any label mappings.
(TagManifestLabelMap
.delete()
.where(TagManifestLabelMap.manifest == manifest_id)
.execute())
# Delete any mapping rows for the manifest.
TagManifestToManifest.delete().where(TagManifestToManifest.manifest == manifest_id).execute()
# Delete any label rows.
ManifestLabel.delete().where(ManifestLabel.manifest == manifest_id,
ManifestLabel.repository == context.repository).execute()
# Delete any child manifest rows.
ManifestChild.delete().where(ManifestChild.manifest == manifest_id,
ManifestChild.repository == context.repository).execute()
# Delete the manifest blobs for the manifest.
ManifestBlob.delete().where(ManifestBlob.manifest == manifest_id,
ManifestBlob.repository == context.repository).execute()
# Delete the manifest legacy image row.
if legacy_image_id:
(ManifestLegacyImage
.delete()
.where(ManifestLegacyImage.manifest == manifest_id,
ManifestLegacyImage.repository == context.repository)
.execute())
# Delete the manifest.
manifest.delete_instance()
context.mark_manifest_removed(manifest)
return True
def _garbage_collect_manifest(manifest_id, context):
assert manifest_id is not None
# Make sure the manifest isn't referenced.
if _check_manifest_used(manifest_id):
return False
# Add the manifest's blobs to the context to be GCed.
for manifest_blob in ManifestBlob.select().where(
ManifestBlob.manifest == manifest_id):
context.add_blob_id(manifest_blob.blob_id)
# Retrieve the manifest's associated image, if any.
try:
legacy_image_id = ManifestLegacyImage.get(
manifest=manifest_id).image_id
context.add_legacy_image_id(legacy_image_id)
except ManifestLegacyImage.DoesNotExist:
legacy_image_id = None
# Add child manifests to be GCed.
for connector in ManifestChild.select().where(
ManifestChild.manifest == manifest_id):
context.add_manifest_id(connector.child_manifest_id)
# Add the labels to be GCed.
for manifest_label in ManifestLabel.select().where(
ManifestLabel.manifest == manifest_id):
context.add_label_id(manifest_label.label_id)
# Delete the manifest.
with db_transaction():
try:
manifest = Manifest.select().where(
Manifest.id == manifest_id).get()
except Manifest.DoesNotExist:
return False
assert manifest.id == manifest_id
assert manifest.repository_id == context.repository.id
if _check_manifest_used(manifest_id):
return False
# Delete any label mappings.
deleted_tag_manifest_label_map = (TagManifestLabelMap.delete().where(
TagManifestLabelMap.manifest == manifest_id).execute())
# Delete any mapping rows for the manifest.
deleted_tag_manifest_to_manifest = (
TagManifestToManifest.delete().where(
TagManifestToManifest.manifest == manifest_id).execute())
# Delete any label rows.
deleted_manifest_label = (ManifestLabel.delete().where(
ManifestLabel.manifest == manifest_id,
ManifestLabel.repository == context.repository,
).execute())
# Delete any child manifest rows.
deleted_manifest_child = (ManifestChild.delete().where(
ManifestChild.manifest == manifest_id,
ManifestChild.repository == context.repository,
).execute())
# Delete the manifest blobs for the manifest.
deleted_manifest_blob = (ManifestBlob.delete().where(
ManifestBlob.manifest == manifest_id,
ManifestBlob.repository == context.repository).execute())
# Delete the security status for the manifest
deleted_manifest_security = (ManifestSecurityStatus.delete().where(
ManifestSecurityStatus.manifest == manifest_id,
ManifestSecurityStatus.repository == context.repository,
).execute())
# Delete the manifest legacy image row.
deleted_manifest_legacy_image = 0
if legacy_image_id:
deleted_manifest_legacy_image = (
ManifestLegacyImage.delete().where(
ManifestLegacyImage.manifest == manifest_id,
ManifestLegacyImage.repository == context.repository,
).execute())
# Delete the manifest.
manifest.delete_instance()
context.mark_manifest_removed(manifest)
gc_table_rows_deleted.labels(
table="TagManifestLabelMap").inc(deleted_tag_manifest_label_map)
gc_table_rows_deleted.labels(
table="TagManifestToManifest").inc(deleted_tag_manifest_to_manifest)
gc_table_rows_deleted.labels(
table="ManifestLabel").inc(deleted_manifest_label)
gc_table_rows_deleted.labels(
table="ManifestChild").inc(deleted_manifest_child)
gc_table_rows_deleted.labels(
table="ManifestBlob").inc(deleted_manifest_blob)
gc_table_rows_deleted.labels(
table="ManifestSecurityStatus").inc(deleted_manifest_security)
if deleted_manifest_legacy_image:
gc_table_rows_deleted.labels(
table="ManifestLegacyImage").inc(deleted_manifest_legacy_image)
gc_table_rows_deleted.labels(table="Manifest").inc()
return True
def _create_manifest(
repository_id,
manifest_interface_instance,
storage,
temp_tag_expiration_sec=TEMP_TAG_EXPIRATION_SEC,
for_tagging=False,
raise_on_error=False,
retriever=None,
):
# Validate the manifest.
retriever = retriever or RepositoryContentRetriever.for_repository(
repository_id, storage)
try:
manifest_interface_instance.validate(retriever)
except (ManifestException, MalformedSchema2ManifestList, BlobDoesNotExist,
IOError) as ex:
logger.exception("Could not validate manifest `%s`",
manifest_interface_instance.digest)
if raise_on_error:
raise CreateManifestException(str(ex))
return None
# Load, parse and get/create the child manifests, if any.
child_manifest_refs = manifest_interface_instance.child_manifests(
retriever)
child_manifest_rows = {}
child_manifest_label_dicts = []
if child_manifest_refs is not None:
for child_manifest_ref in child_manifest_refs:
# Load and parse the child manifest.
try:
child_manifest = child_manifest_ref.manifest_obj
except (
ManifestException,
MalformedSchema2ManifestList,
BlobDoesNotExist,
IOError,
) as ex:
logger.exception(
"Could not load manifest list for manifest `%s`",
manifest_interface_instance.digest,
)
if raise_on_error:
raise CreateManifestException(str(ex))
return None
# Retrieve its labels.
labels = child_manifest.get_manifest_labels(retriever)
if labels is None:
if raise_on_error:
raise CreateManifestException(
"Unable to retrieve manifest labels")
logger.exception(
"Could not load manifest labels for child manifest")
return None
# Get/create the child manifest in the database.
child_manifest_info = get_or_create_manifest(
repository_id,
child_manifest,
storage,
raise_on_error=raise_on_error)
if child_manifest_info is None:
if raise_on_error:
raise CreateManifestException(
"Unable to retrieve child manifest")
logger.error("Could not get/create child manifest")
return None
child_manifest_rows[child_manifest_info.manifest.
digest] = child_manifest_info.manifest
child_manifest_label_dicts.append(labels)
# Build the map from required blob digests to the blob objects.
blob_map = _build_blob_map(
repository_id,
manifest_interface_instance,
retriever,
storage,
raise_on_error,
require_empty_layer=False,
)
if blob_map is None:
return None
# Create the manifest and its blobs.
media_type = Manifest.media_type.get_id(
manifest_interface_instance.media_type)
storage_ids = {storage.id for storage in list(blob_map.values())}
# Check for the manifest, in case it was created since we checked earlier.
try:
manifest = Manifest.get(repository=repository_id,
digest=manifest_interface_instance.digest)
return CreatedManifest(manifest=manifest,
newly_created=False,
labels_to_apply=None)
except Manifest.DoesNotExist:
pass
try:
with db_transaction():
# Create the manifest.
try:
manifest = Manifest.create(
repository=repository_id,
digest=manifest_interface_instance.digest,
media_type=media_type,
manifest_bytes=manifest_interface_instance.bytes.
as_encoded_str(),
config_media_type=manifest_interface_instance.
config_media_type,
layers_compressed_size=manifest_interface_instance.
layers_compressed_size,
)
except IntegrityError as ie:
# NOTE: An IntegrityError means (barring a bug) that the manifest was created by
# another caller while we were attempting to create it. Since we need to return
# the manifest, we raise a specialized exception here to break out of the
# transaction so we can retrieve it.
raise _ManifestAlreadyExists(ie)
# Insert the blobs.
blobs_to_insert = [
dict(manifest=manifest,
repository=repository_id,
blob=storage_id) for storage_id in storage_ids
]
if blobs_to_insert:
try:
ManifestBlob.insert_many(blobs_to_insert).execute()
except IntegrityError as ie:
raise _ManifestAlreadyExists(ie)
# Insert the manifest child rows (if applicable).
if child_manifest_rows:
children_to_insert = [
dict(manifest=manifest,
child_manifest=child_manifest,
repository=repository_id)
for child_manifest in list(child_manifest_rows.values())
]
try:
ManifestChild.insert_many(children_to_insert).execute()
except IntegrityError as ie:
raise _ManifestAlreadyExists(ie)
# If this manifest is being created not for immediate tagging, add a temporary tag to the
# manifest to ensure it isn't being GCed. If the manifest *is* for tagging, then since we're
# creating a new one here, it cannot be GCed (since it isn't referenced by anything yet), so
# its safe to elide the temp tag operation. If we ever change GC code to collect *all* manifests
# in a repository for GC, then we will have to reevaluate this optimization at that time.
if not for_tagging:
create_temporary_tag_if_necessary(manifest,
temp_tag_expiration_sec)
# Define the labels for the manifest (if any).
# TODO: Once the old data model is gone, turn this into a batch operation and make the label
# application to the manifest occur under the transaction.
labels = manifest_interface_instance.get_manifest_labels(retriever)
if labels:
for key, value in labels.items():
# NOTE: There can technically be empty label keys via Dockerfile's. We ignore any
# such `labels`, as they don't really mean anything.
if not key:
continue
media_type = "application/json" if is_json(
value) else "text/plain"
create_manifest_label(manifest, key, value, "manifest",
media_type)
# Return the dictionary of labels to apply (i.e. those labels that cause an action to be taken
# on the manifest or its resulting tags). We only return those labels either defined on
# the manifest or shared amongst all the child manifests. We intersect amongst all child manifests
# to ensure that any action performed is defined in all manifests.
labels_to_apply = labels or {}
if child_manifest_label_dicts:
labels_to_apply = child_manifest_label_dicts[0].items()
for child_manifest_label_dict in child_manifest_label_dicts[1:]:
# Intersect the key+values of the labels to ensure we get the exact same result
# for all the child manifests.
labels_to_apply = labels_to_apply & child_manifest_label_dict.items(
)
labels_to_apply = dict(labels_to_apply)
return CreatedManifest(manifest=manifest,
newly_created=True,
labels_to_apply=labels_to_apply)
except _ManifestAlreadyExists as mae:
try:
manifest = Manifest.get(repository=repository_id,
digest=manifest_interface_instance.digest)
except Manifest.DoesNotExist:
# NOTE: If we've reached this point, then somehow we had an IntegrityError without it
# being due to a duplicate manifest. We therefore log the error.
logger.error(
"Got integrity error when trying to create manifest: %s",
mae.internal_exception)
if raise_on_error:
raise CreateManifestException(
"Attempt to create an invalid manifest. Please report this issue."
)
return None
return CreatedManifest(manifest=manifest,
newly_created=False,
labels_to_apply=None)
def _create_manifest(
repository_id,
manifest_interface_instance,
storage,
temp_tag_expiration_sec=TEMP_TAG_EXPIRATION_SEC,
for_tagging=False,
raise_on_error=False,
retriever=None,
):
# Validate the manifest.
retriever = retriever or RepositoryContentRetriever.for_repository(
repository_id, storage)
try:
manifest_interface_instance.validate(retriever)
except (ManifestException, MalformedSchema2ManifestList, BlobDoesNotExist,
IOError) as ex:
logger.exception("Could not validate manifest `%s`",
manifest_interface_instance.digest)
if raise_on_error:
raise CreateManifestException(str(ex))
return None
# Load, parse and get/create the child manifests, if any.
child_manifest_refs = manifest_interface_instance.child_manifests(
retriever)
child_manifest_rows = {}
child_manifest_label_dicts = []
if child_manifest_refs is not None:
for child_manifest_ref in child_manifest_refs:
# Load and parse the child manifest.
try:
child_manifest = child_manifest_ref.manifest_obj
except (
ManifestException,
MalformedSchema2ManifestList,
BlobDoesNotExist,
IOError,
) as ex:
logger.exception(
"Could not load manifest list for manifest `%s`",
manifest_interface_instance.digest,
)
if raise_on_error:
raise CreateManifestException(str(ex))
return None
# Retrieve its labels.
labels = child_manifest.get_manifest_labels(retriever)
if labels is None:
if raise_on_error:
raise CreateManifestException(
"Unable to retrieve manifest labels")
logger.exception(
"Could not load manifest labels for child manifest")
return None
# Get/create the child manifest in the database.
child_manifest_info = get_or_create_manifest(
repository_id,
child_manifest,
storage,
raise_on_error=raise_on_error)
if child_manifest_info is None:
if raise_on_error:
raise CreateManifestException(
"Unable to retrieve child manifest")
logger.error("Could not get/create child manifest")
return None
child_manifest_rows[child_manifest_info.manifest.
digest] = child_manifest_info.manifest
child_manifest_label_dicts.append(labels)
# Ensure all the blobs in the manifest exist.
digests = set(manifest_interface_instance.local_blob_digests)
blob_map = {}
# If the special empty layer is required, simply load it directly. This is much faster
# than trying to load it on a per repository basis, and that is unnecessary anyway since
# this layer is predefined.
if EMPTY_LAYER_BLOB_DIGEST in digests:
digests.remove(EMPTY_LAYER_BLOB_DIGEST)
blob_map[EMPTY_LAYER_BLOB_DIGEST] = get_shared_blob(
EMPTY_LAYER_BLOB_DIGEST)
if not blob_map[EMPTY_LAYER_BLOB_DIGEST]:
if raise_on_error:
raise CreateManifestException(
"Unable to retrieve specialized empty blob")
logger.warning("Could not find the special empty blob in storage")
return None
if digests:
query = lookup_repo_storages_by_content_checksum(
repository_id, digests)
blob_map.update({s.content_checksum: s for s in query})
for digest_str in digests:
if digest_str not in blob_map:
logger.warning(
"Unknown blob `%s` under manifest `%s` for repository `%s`",
digest_str,
manifest_interface_instance.digest,
repository_id,
)
if raise_on_error:
raise CreateManifestException("Unknown blob `%s`" %
digest_str)
return None
# Special check: If the empty layer blob is needed for this manifest, add it to the
# blob map. This is necessary because Docker decided to elide sending of this special
# empty layer in schema version 2, but we need to have it referenced for GC and schema version 1.
if EMPTY_LAYER_BLOB_DIGEST not in blob_map:
try:
requires_empty_layer = manifest_interface_instance.get_requires_empty_layer_blob(
retriever)
except ManifestException as ex:
if raise_on_error:
raise CreateManifestException(str(ex))
return None
if requires_empty_layer is None:
if raise_on_error:
raise CreateManifestException(
"Could not load configuration blob")
return None
if requires_empty_layer:
shared_blob = get_or_create_shared_blob(EMPTY_LAYER_BLOB_DIGEST,
EMPTY_LAYER_BYTES, storage)
assert not shared_blob.uploading
assert shared_blob.content_checksum == EMPTY_LAYER_BLOB_DIGEST
blob_map[EMPTY_LAYER_BLOB_DIGEST] = shared_blob
# Determine and populate the legacy image if necessary. Manifest lists will not have a legacy
# image.
legacy_image = None
if manifest_interface_instance.has_legacy_image:
legacy_image_id = _populate_legacy_image(repository_id,
manifest_interface_instance,
blob_map, retriever,
raise_on_error)
if legacy_image_id is None:
return None
legacy_image = get_image(repository_id, legacy_image_id)
if legacy_image is None:
return None
# Create the manifest and its blobs.
media_type = Manifest.media_type.get_id(
manifest_interface_instance.media_type)
storage_ids = {storage.id for storage in blob_map.values()}
with db_transaction():
# Check for the manifest. This is necessary because Postgres doesn't handle IntegrityErrors
# well under transactions.
try:
manifest = Manifest.get(repository=repository_id,
digest=manifest_interface_instance.digest)
return CreatedManifest(manifest=manifest,
newly_created=False,
labels_to_apply=None)
except Manifest.DoesNotExist:
pass
# Create the manifest.
try:
manifest = Manifest.create(
repository=repository_id,
digest=manifest_interface_instance.digest,
media_type=media_type,
manifest_bytes=manifest_interface_instance.bytes.
as_encoded_str(),
)
except IntegrityError as ie:
try:
manifest = Manifest.get(
repository=repository_id,
digest=manifest_interface_instance.digest)
except Manifest.DoesNotExist:
logger.error(
"Got integrity error when trying to create manifest: %s",
ie)
if raise_on_error:
raise CreateManifestException(
"Attempt to create an invalid manifest. Please report this issue."
)
return None
return CreatedManifest(manifest=manifest,
newly_created=False,
labels_to_apply=None)
# Insert the blobs.
blobs_to_insert = [
dict(manifest=manifest, repository=repository_id, blob=storage_id)
for storage_id in storage_ids
]
if blobs_to_insert:
ManifestBlob.insert_many(blobs_to_insert).execute()
# Set the legacy image (if applicable).
if legacy_image is not None:
ManifestLegacyImage.create(repository=repository_id,
image=legacy_image,
manifest=manifest)
# Insert the manifest child rows (if applicable).
if child_manifest_rows:
children_to_insert = [
dict(manifest=manifest,
child_manifest=child_manifest,
repository=repository_id)
for child_manifest in child_manifest_rows.values()
]
ManifestChild.insert_many(children_to_insert).execute()
# If this manifest is being created not for immediate tagging, add a temporary tag to the
# manifest to ensure it isn't being GCed. If the manifest *is* for tagging, then since we're
# creating a new one here, it cannot be GCed (since it isn't referenced by anything yet), so
# its safe to elide the temp tag operation. If we ever change GC code to collect *all* manifests
# in a repository for GC, then we will have to reevaluate this optimization at that time.
if not for_tagging:
create_temporary_tag_if_necessary(manifest,
temp_tag_expiration_sec)
# Define the labels for the manifest (if any).
# TODO: Once the old data model is gone, turn this into a batch operation and make the label
# application to the manifest occur under the transaction.
labels = manifest_interface_instance.get_manifest_labels(retriever)
if labels:
for key, value in labels.iteritems():
# NOTE: There can technically be empty label keys via Dockerfile's. We ignore any
# such `labels`, as they don't really mean anything.
if not key:
continue
media_type = "application/json" if is_json(value) else "text/plain"
create_manifest_label(manifest, key, value, "manifest", media_type)
# Return the dictionary of labels to apply (i.e. those labels that cause an action to be taken
# on the manifest or its resulting tags). We only return those labels either defined on
# the manifest or shared amongst all the child manifests. We intersect amongst all child manifests
# to ensure that any action performed is defined in all manifests.
labels_to_apply = labels or {}
if child_manifest_label_dicts:
labels_to_apply = child_manifest_label_dicts[0].viewitems()
for child_manifest_label_dict in child_manifest_label_dicts[1:]:
# Intersect the key+values of the labels to ensure we get the exact same result
# for all the child manifests.
labels_to_apply = labels_to_apply & child_manifest_label_dict.viewitems(
)
labels_to_apply = dict(labels_to_apply)
return CreatedManifest(manifest=manifest,
newly_created=True,
labels_to_apply=labels_to_apply)
def test_get_or_create_manifest_list_duplicate_child_manifest(initialized_db):
repository = create_repository("devtable", "newrepo", None)
expected_labels = {
"Foo": "Bar",
"Baz": "Meh",
}
layer_json = json.dumps({
"id":
"somelegacyid",
"config": {
"Labels": expected_labels,
},
"rootfs": {
"type": "layers",
"diff_ids": []
},
"history": [
{
"created": "2018-04-03T18:37:09.284840891Z",
"created_by": "do something",
},
],
})
# Create a legacy image.
find_create_or_link_image("somelegacyid", repository, "devtable", {},
"local_us")
# Add a blob containing the config.
_, config_digest = _populate_blob(layer_json)
# Add a blob of random data.
random_data = "hello world"
_, random_digest = _populate_blob(random_data)
# Build the manifest.
v2_builder = DockerSchema2ManifestBuilder()
v2_builder.set_config_digest(config_digest,
len(layer_json.encode("utf-8")))
v2_builder.add_layer(random_digest, len(random_data.encode("utf-8")))
v2_manifest = v2_builder.build()
# Write the manifest.
v2_created = get_or_create_manifest(repository, v2_manifest, storage)
assert v2_created
assert v2_created.manifest.digest == v2_manifest.digest
# Build the manifest list, with the child manifest repeated.
list_builder = DockerSchema2ManifestListBuilder()
list_builder.add_manifest(v2_manifest, "amd64", "linux")
list_builder.add_manifest(v2_manifest, "amd32", "linux")
manifest_list = list_builder.build()
# Write the manifest list, which should also write the manifests themselves.
created_tuple = get_or_create_manifest(repository, manifest_list, storage)
assert created_tuple is not None
created_list = created_tuple.manifest
assert created_list
assert created_list.media_type.name == manifest_list.media_type
assert created_list.digest == manifest_list.digest
# Ensure the child manifest links exist.
child_manifests = {
cm.child_manifest.digest: cm.child_manifest
for cm in ManifestChild.select().where(
ManifestChild.manifest == created_list)
}
assert len(child_manifests) == 1
assert v2_manifest.digest in child_manifests
assert child_manifests[
v2_manifest.digest].media_type.name == v2_manifest.media_type
# Try to create again and ensure we get back the same manifest list.
created2_tuple = get_or_create_manifest(repository, manifest_list, storage)
assert created2_tuple is not None
assert created2_tuple.manifest == created_list
def _create_manifest_and_retarget_tag(
self, repository_ref: RepositoryReference,
manifest: ManifestInterface,
tag_name: str) -> tuple[Manifest | None, Tag | None]:
"""
Creates a manifest in the given repository.
Also creates placeholders for the objects referenced by the manifest.
For manifest lists, creates placeholder sub-manifests. For regular
manifests, creates placeholder blobs.
Placeholder objects will be "filled" with the objects' contents on
upcoming client requests, as part of the flow described in the OCI
distribution specification.
Returns a reference to the (created manifest, tag) or (None, None) on error.
"""
with db_disallow_replica_use():
with db_transaction():
db_manifest = oci.manifest.lookup_manifest(repository_ref.id,
manifest.digest,
allow_dead=True)
if db_manifest is None:
db_manifest = oci.manifest.create_manifest(
repository_ref.id, manifest, raise_on_error=True)
self._recalculate_repository_size(repository_ref)
if db_manifest is None:
return None, None
# 0 means a tag never expires - if we get 0 as expiration,
# we set the tag expiration to None.
expiration = self._config.expiration_s or None
tag = oci.tag.retarget_tag(
tag_name,
db_manifest,
raise_on_error=True,
expiration_seconds=expiration,
)
if tag is None:
return None, None
wrapped_manifest = Manifest.for_manifest(
db_manifest, self._legacy_image_id_handler)
wrapped_tag = Tag.for_tag(tag,
self._legacy_image_id_handler,
manifest_row=db_manifest)
if not manifest.is_manifest_list:
self._create_placeholder_blobs(manifest, db_manifest.id,
repository_ref.id)
return wrapped_manifest, wrapped_tag
manifests_to_connect = []
for child in manifest.child_manifests(content_retriever=None):
m = oci.manifest.lookup_manifest(repository_ref.id,
child.digest,
allow_dead=True)
if m is None:
m = oci.manifest.create_manifest(
repository_ref.id, child)
oci.tag.create_temporary_tag_if_necessary(
m, self._config.expiration_s or None)
try:
ManifestChild.get(manifest=db_manifest.id,
child_manifest=m.id)
except ManifestChild.DoesNotExist:
manifests_to_connect.append(m)
oci.manifest.connect_manifests(manifests_to_connect,
db_manifest, repository_ref.id)
return wrapped_manifest, wrapped_tag
def lookup_manifest_by_digest(
self,
repository_ref,
manifest_digest,
allow_dead=False,
require_available=False,
raise_on_error=True,
):
"""
Looks up the manifest with the given digest under the given repository and returns it or
None if none.
If a manifest with the digest does not exist, fetches the manifest upstream
and creates it with a temp tag.
"""
wrapped_manifest = super().lookup_manifest_by_digest(
repository_ref,
manifest_digest,
allow_dead=True,
require_available=False)
if wrapped_manifest is None:
try:
wrapped_manifest, _ = self._create_and_tag_manifest(
repository_ref, manifest_digest,
self._create_manifest_with_temp_tag)
except (UpstreamRegistryError, ManifestDoesNotExist) as e:
raise ManifestDoesNotExist(str(e))
return wrapped_manifest
db_tag = oci.tag.get_tag_by_manifest_id(repository_ref.id,
wrapped_manifest.id)
existing_tag = Tag.for_tag(db_tag,
self._legacy_image_id_handler,
manifest_row=db_tag.manifest)
new_tag = False
try:
tag, new_tag = self._update_manifest_for_tag(
repository_ref,
existing_tag,
existing_tag.manifest,
manifest_digest,
self._create_manifest_with_temp_tag,
)
except ManifestDoesNotExist as e:
raise e
except UpstreamRegistryError:
# when the upstream fetch fails, we only return the tag if
# it isn't yet expired. note that we don't bump the tag's
# expiration here either - we only do this when we can ensure
# the tag exists upstream.
isplaceholder = wrapped_manifest.internal_manifest_bytes.as_unicode(
) == ""
return wrapped_manifest if not existing_tag.expired and not isplaceholder else None
if tag.expired or not new_tag:
with db_disallow_replica_use():
new_expiration = (get_epoch_timestamp_ms() +
self._config.expiration_s * 1000
if self._config.expiration_s else None)
oci.tag.set_tag_end_ms(db_tag, new_expiration)
# if the manifest is a child of a manifest list in this repo, renew
# the parent manifest list tag too.
parent = ManifestChild.select(ManifestChild.manifest_id).where(
(ManifestChild.repository_id == repository_ref.id)
& (ManifestChild.child_manifest_id == wrapped_manifest.id))
parent_tag = oci.tag.get_tag_by_manifest_id(
repository_ref.id, parent)
if parent_tag is not None:
oci.tag.set_tag_end_ms(parent_tag, new_expiration)
return super().lookup_manifest_by_digest(
repository_ref,
manifest_digest,
allow_dead=True,
require_available=False,
raise_on_error=True,
)
def test_manifest_list_create_manifest_with_sub_manifests_and_connect_them(
self, test_name, proxy_manifest_response):
test_params = storage_test_cases[test_name]
if test_params["manifest_type"] not in [
DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE,
OCI_IMAGE_INDEX_CONTENT_TYPE,
]:
pytest.skip(
"regular manifest detected, skipping manifest list specific test."
)
repo = f"{self.orgname}/{test_params['image_name']}"
params = {
"repository": repo,
"manifest_ref": test_params["manifest_ref"],
}
proxy_mock = proxy_manifest_response(
test_params["manifest_ref"],
test_params["manifest_json"],
test_params["manifest_type"],
)
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
headers = _get_auth_headers(self.sub, self.ctx, repo)
headers["Accept"] = ", ".join(
DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union(
DOCKER_SCHEMA1_CONTENT_TYPES))
conduct_call(
self.client,
test_params["view_name"],
url_for,
"GET",
params,
expected_code=200,
headers=headers,
)
manifest_list = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(test_params["manifest_json"]),
test_params["manifest_type"],
sparse_manifest_support=True,
)
try:
manifest = Manifest.filter(
Manifest.digest == manifest_list.digest).get()
except Manifest.DoesNotExist:
assert False, "failed to create manifest list"
input_digests = [
manifest["digest"]
for manifest in manifest_list.manifest_dict["manifests"]
]
manifest_links = ManifestChild.select(
ManifestChild.child_manifest).where(
ManifestChild.manifest == manifest)
sub_digests = [ml.child_manifest.digest for ml in manifest_links]
assert input_digests == sub_digests
for link in manifest_links:
mbytes = link.child_manifest.manifest_bytes
assert mbytes == "", f"child manifest bytes expected empty, but got {mbytes}"
def test_get_or_create_manifest_list(initialized_db):
repository = create_repository('devtable', 'newrepo', None)
expected_labels = {
'Foo': 'Bar',
'Baz': 'Meh',
}
layer_json = json.dumps({
'id':
'somelegacyid',
'config': {
'Labels': expected_labels,
},
"rootfs": {
"type": "layers",
"diff_ids": []
},
"history": [
{
"created": "2018-04-03T18:37:09.284840891Z",
"created_by": "do something",
},
],
})
# Create a legacy image.
find_create_or_link_image('somelegacyid', repository, 'devtable', {},
'local_us')
# Add a blob containing the config.
_, config_digest = _populate_blob(layer_json)
# Add a blob of random data.
random_data = 'hello world'
_, random_digest = _populate_blob(random_data)
# Build the manifests.
v1_builder = DockerSchema1ManifestBuilder('devtable', 'simple',
'anothertag')
v1_builder.add_layer(random_digest, layer_json)
v1_manifest = v1_builder.build(docker_v2_signing_key).unsigned()
v2_builder = DockerSchema2ManifestBuilder()
v2_builder.set_config_digest(config_digest, len(layer_json))
v2_builder.add_layer(random_digest, len(random_data))
v2_manifest = v2_builder.build()
# Write the manifests.
v1_created = get_or_create_manifest(repository, v1_manifest, storage)
assert v1_created
assert v1_created.manifest.digest == v1_manifest.digest
v2_created = get_or_create_manifest(repository, v2_manifest, storage)
assert v2_created
assert v2_created.manifest.digest == v2_manifest.digest
# Build the manifest list.
list_builder = DockerSchema2ManifestListBuilder()
list_builder.add_manifest(v1_manifest, 'amd64', 'linux')
list_builder.add_manifest(v2_manifest, 'amd32', 'linux')
manifest_list = list_builder.build()
# Write the manifest list, which should also write the manifests themselves.
created_tuple = get_or_create_manifest(repository, manifest_list, storage)
assert created_tuple is not None
created_list = created_tuple.manifest
assert created_list
assert created_list.media_type.name == manifest_list.media_type
assert created_list.digest == manifest_list.digest
# Ensure the child manifest links exist.
child_manifests = {
cm.child_manifest.digest: cm.child_manifest
for cm in ManifestChild.select().where(
ManifestChild.manifest == created_list)
}
assert len(child_manifests) == 2
assert v1_manifest.digest in child_manifests
assert v2_manifest.digest in child_manifests
assert child_manifests[
v1_manifest.digest].media_type.name == v1_manifest.media_type
assert child_manifests[
v2_manifest.digest].media_type.name == v2_manifest.media_type
