def delete_manifests():
ManifestLegacyImage.delete().execute()
ManifestBlob.delete().execute()
Manifest.delete().execute()
TagManifestToManifest.delete().execute()
TagManifest.delete().execute()
return "OK"
def populate_manifest(repository, manifest, legacy_image, storage_ids):
""" Populates the rows for the manifest, including its blobs and legacy image. """
media_type = Manifest.media_type.get_id(manifest.media_type)
# Check for an existing manifest. If present, return it.
try:
return Manifest.get(repository=repository, digest=manifest.digest)
except Manifest.DoesNotExist:
pass
with db_transaction():
try:
manifest_row = Manifest.create(
digest=manifest.digest,
repository=repository,
manifest_bytes=manifest.bytes.as_encoded_str(),
media_type=media_type,
)
except IntegrityError as ie:
logger.debug(
"Got integrity error when trying to write manifest: %s", ie)
return Manifest.get(repository=repository, digest=manifest.digest)
ManifestLegacyImage.create(manifest=manifest_row,
repository=repository,
image=legacy_image)
blobs_to_insert = [
dict(manifest=manifest_row, repository=repository, blob=storage_id)
for storage_id in storage_ids
]
if blobs_to_insert:
ManifestBlob.insert_many(blobs_to_insert).execute()
return manifest_row
def _check_image_used(legacy_image_id):
assert legacy_image_id is not None
with db_transaction():
# Check if the image is referenced by a manifest.
try:
ManifestLegacyImage.select().where(ManifestLegacyImage.image == legacy_image_id).get()
return True
except ManifestLegacyImage.DoesNotExist:
pass
# Check if the image is referenced by a tag.
try:
RepositoryTag.select().where(RepositoryTag.image == legacy_image_id).get()
return True
except RepositoryTag.DoesNotExist:
pass
# Check if the image is referenced by another image.
try:
Image.select().where(Image.parent == legacy_image_id).get()
return True
except Image.DoesNotExist:
pass
return False
def clear_rows(initialized_db):
# Remove all new-style rows so we can backfill.
TagToRepositoryTag.delete().execute()
Tag.delete().execute()
TagManifestLabelMap.delete().execute()
ManifestLabel.delete().execute()
ManifestBlob.delete().execute()
ManifestLegacyImage.delete().execute()
TagManifestToManifest.delete().execute()
Manifest.delete().execute()
def test_manifestbackfillworker_mislinked_invalid_manifest(clear_rows, initialized_db):
""" Tests that a manifest whose image is mislinked will attempt to have its storages relinked
properly. """
# Delete existing tag manifest so we can reuse the tag.
TagManifestLabel.delete().execute()
TagManifest.delete().execute()
repo = model.repository.get_repository("devtable", "complex")
tag_v50 = model.tag.get_active_tag("devtable", "gargantuan", "v5.0")
# Add a mislinked manifest, by having its layer point to an invalid blob but its image
# be the v5.0 image.
builder = DockerSchema1ManifestBuilder("devtable", "gargantuan", "sometag")
builder.add_layer("sha256:deadbeef", '{"id": "foo"}')
manifest = builder.build(docker_v2_signing_key)
broken_manifest = TagManifest.create(
json_data=manifest.bytes.as_encoded_str(), digest=manifest.digest, tag=tag_v50
)
# Backfill the manifest and ensure it is marked as broken.
assert _backfill_manifest(broken_manifest)
map_row = TagManifestToManifest.get(tag_manifest=broken_manifest)
assert map_row.broken
manifest_row = map_row.manifest
legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image
assert legacy_image == tag_v50.image
manifest_blobs = list(ManifestBlob.select().where(ManifestBlob.manifest == manifest_row))
assert len(manifest_blobs) == 0
def test_store_tag_manifest(get_storages, initialized_db):
# Create a manifest with some layers.
builder = DockerSchema1ManifestBuilder('devtable', 'simple', 'sometag')
storages = get_storages()
assert storages
repo = model.repository.get_repository('devtable', 'simple')
storage_id_map = {}
for index, storage in enumerate(storages):
image_id = 'someimage%s' % index
builder.add_layer(storage.content_checksum, json.dumps({'id': image_id}))
find_create_or_link_image(image_id, repo, 'devtable', {}, 'local_us')
storage_id_map[storage.content_checksum] = storage.id
manifest = builder.build(docker_v2_signing_key)
tag_manifest, _ = store_tag_manifest_for_testing('devtable', 'simple', 'sometag', manifest,
manifest.leaf_layer_v1_image_id, storage_id_map)
# Ensure we have the new-model expected rows.
mapping_row = TagManifestToManifest.get(tag_manifest=tag_manifest)
assert mapping_row.manifest is not None
assert mapping_row.manifest.manifest_bytes == manifest.bytes.as_encoded_str()
assert mapping_row.manifest.digest == str(manifest.digest)
blob_rows = {m.blob_id for m in
ManifestBlob.select().where(ManifestBlob.manifest == mapping_row.manifest)}
assert blob_rows == {s.id for s in storages}
assert ManifestLegacyImage.get(manifest=mapping_row.manifest).image == tag_manifest.tag.image
def test_manifestbackfillworker_mislinked_manifest(clear_rows, initialized_db):
""" Tests that a manifest whose image is mislinked will have its storages relinked properly. """
# Delete existing tag manifest so we can reuse the tag.
TagManifestLabel.delete().execute()
TagManifest.delete().execute()
repo = model.repository.get_repository('devtable', 'complex')
tag_v30 = model.tag.get_active_tag('devtable', 'gargantuan', 'v3.0')
tag_v50 = model.tag.get_active_tag('devtable', 'gargantuan', 'v5.0')
# Add a mislinked manifest, by having its layer point to a blob in v3.0 but its image
# be the v5.0 image.
builder = DockerSchema1ManifestBuilder('devtable', 'gargantuan', 'sometag')
builder.add_layer(tag_v30.image.storage.content_checksum, '{"id": "foo"}')
manifest = builder.build(docker_v2_signing_key)
mislinked_manifest = TagManifest.create(json_data=manifest.bytes.as_encoded_str(),
digest=manifest.digest,
tag=tag_v50)
# Backfill the manifest and ensure its proper content checksum was linked.
assert _backfill_manifest(mislinked_manifest)
map_row = TagManifestToManifest.get(tag_manifest=mislinked_manifest)
assert not map_row.broken
manifest_row = map_row.manifest
legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image
assert legacy_image == tag_v50.image
manifest_blobs = list(ManifestBlob.select().where(ManifestBlob.manifest == manifest_row))
assert len(manifest_blobs) == 1
assert manifest_blobs[0].blob.content_checksum == tag_v30.image.storage.content_checksum
def get_legacy_image_for_manifest(manifest_id):
""" Returns the legacy image associated with the given manifest, if any, or None if none. """
try:
query = (ManifestLegacyImage.select(
ManifestLegacyImage, Image).join(Image).where(
ManifestLegacyImage.manifest == manifest_id))
return query.get().image
except ManifestLegacyImage.DoesNotExist:
return None
def get_manifest_for_legacy_image(image_id):
""" Returns a manifest that is associated with the given image, if any, or None if none. """
try:
query = (ManifestLegacyImage.select(
ManifestLegacyImage, Manifest).join(Manifest).where(
ManifestLegacyImage.image == image_id))
return query.get().manifest
except ManifestLegacyImage.DoesNotExist:
return None
def populate_legacy_images_for_testing(manifest, manifest_interface_instance,
storage):
""" Populates the legacy image rows for the given manifest. """
# NOTE: This method is only kept around for use by legacy tests that still require
# legacy images. As a result, we make sure we're in testing mode before we run.
assert os.getenv("TEST") == "true"
repository_id = manifest.repository_id
retriever = RepositoryContentRetriever.for_repository(
repository_id, storage)
blob_map = _build_blob_map(repository_id,
manifest_interface_instance,
storage,
True,
require_empty_layer=True)
if blob_map is None:
return None
# Determine and populate the legacy image if necessary. Manifest lists will not have a legacy
# image.
legacy_image = None
if manifest_interface_instance.has_legacy_image:
try:
legacy_image_id = _populate_legacy_image(
repository_id, manifest_interface_instance, blob_map,
retriever, True)
except ManifestException as me:
raise CreateManifestException(
"Attempt to create an invalid manifest: %s. Please report this issue."
% me)
if legacy_image_id is None:
return None
legacy_image = get_image(repository_id, legacy_image_id)
if legacy_image is None:
return None
# Set the legacy image (if applicable).
if legacy_image is not None:
ManifestLegacyImage.create(repository=repository_id,
image=legacy_image,
manifest=manifest)
def verify_backfill(namespace_name):
logger.info('Checking namespace %s', namespace_name)
namespace_user = model.user.get_namespace_user(namespace_name)
assert namespace_user
repo_tags = (RepositoryTag
.select()
.join(Repository)
.where(Repository.namespace_user == namespace_user)
.where(RepositoryTag.hidden == False))
repo_tags = list(repo_tags)
logger.info('Found %s tags', len(repo_tags))
for index, repo_tag in enumerate(repo_tags):
logger.info('Checking tag %s under repository %s (%s/%s)', repo_tag.name,
repo_tag.repository.name, index + 1, len(repo_tags))
tag = TagToRepositoryTag.get(repository_tag=repo_tag).tag
assert not tag.hidden
assert tag.repository == repo_tag.repository
assert tag.name == repo_tag.name, _vs(tag.name, repo_tag.name)
assert tag.repository == repo_tag.repository, _vs(tag.repository_id, repo_tag.repository_id)
assert tag.reversion == repo_tag.reversion, _vs(tag.reversion, repo_tag.reversion)
start_check = int(tag.lifetime_start_ms / 1000) == repo_tag.lifetime_start_ts
assert start_check, _vs(tag.lifetime_start_ms, repo_tag.lifetime_start_ts)
if repo_tag.lifetime_end_ts is not None:
end_check = int(tag.lifetime_end_ms / 1000) == repo_tag.lifetime_end_ts
assert end_check, _vs(tag.lifetime_end_ms, repo_tag.lifetime_end_ts)
else:
assert tag.lifetime_end_ms is None
try:
tag_manifest = tag.manifest
repo_tag_manifest = TagManifest.get(tag=repo_tag)
digest_check = tag_manifest.digest == repo_tag_manifest.digest
assert digest_check, _vs(tag_manifest.digest, repo_tag_manifest.digest)
bytes_check = tag_manifest.manifest_bytes == repo_tag_manifest.json_data
assert bytes_check, _vs(tag_manifest.manifest_bytes, repo_tag_manifest.json_data)
except TagManifest.DoesNotExist:
logger.info('No tag manifest found for repository tag %s', repo_tag.id)
mli = ManifestLegacyImage.get(manifest=tag_manifest)
assert mli.repository == repo_tag.repository
manifest_legacy_image = mli.image
assert manifest_legacy_image == repo_tag.image, _vs(manifest_legacy_image.id, repo_tag.image_id)
def get_legacy_images_for_tags(tags):
""" Returns a map from tag ID to the legacy image for the tag. """
if not tags:
return {}
query = (ManifestLegacyImage.select(
ManifestLegacyImage, Image,
ImageStorage).join(Image).join(ImageStorage).where(
ManifestLegacyImage.manifest << [tag.manifest_id for tag in tags]))
by_manifest = {mli.manifest_id: mli.image for mli in query}
return {
tag.id: by_manifest[tag.manifest_id]
for tag in tags if tag.manifest_id in by_manifest
}
def compute_layer_id(layer):
"""
Returns the ID for the layer in the security scanner.
"""
assert isinstance(layer, ManifestDataType)
manifest = Manifest.get(id=layer._db_id)
try:
layer = ManifestLegacyImage.get(manifest=manifest).image
except ManifestLegacyImage.DoesNotExist:
return None
assert layer.docker_image_id
assert layer.storage.uuid
return "%s.%s" % (layer.docker_image_id, layer.storage.uuid)
def test_load_security_information(indexed_v2, indexed_v4, expected_status,
initialized_db):
secscan_model.configure(app, instance_keys, storage)
repository_ref = registry_model.lookup_repository("devtable", "simple")
tag = registry_model.find_matching_tag(repository_ref, ["latest"])
manifest = registry_model.get_manifest_for_tag(tag)
assert manifest
registry_model.populate_legacy_images_for_testing(manifest, storage)
image = shared.get_legacy_image_for_manifest(manifest._db_id)
if indexed_v2:
image.security_indexed = False
image.security_indexed_engine = 3
image.save()
else:
ManifestLegacyImage.delete().where(
ManifestLegacyImage.manifest == manifest._db_id).execute()
if indexed_v4:
ManifestSecurityStatus.create(
manifest=manifest._db_id,
repository=repository_ref._db_id,
error_json={},
index_status=IndexStatus.MANIFEST_UNSUPPORTED,
indexer_hash="abc",
indexer_version=IndexerVersion.V4,
metadata_json={},
)
result = secscan_model.load_security_information(manifest, True)
assert isinstance(result, SecurityInformationLookupResult)
assert result.status == expected_status
def compute_layer_id(layer):
""" Returns the ID for the layer in the security scanner. """
# NOTE: this is temporary until we switch to Clair V3.
if isinstance(layer, ManifestDataType):
if layer._is_tag_manifest:
layer = TagManifest.get(id=layer._db_id).tag.image
else:
manifest = Manifest.get(id=layer._db_id)
try:
layer = ManifestLegacyImage.get(manifest=manifest).image
except ManifestLegacyImage.DoesNotExist:
return None
elif isinstance(layer, LegacyImage):
layer = Image.get(id=layer._db_id)
assert layer.docker_image_id
assert layer.storage.uuid
return '%s.%s' % (layer.docker_image_id, layer.storage.uuid)
def test_manifestbackfillworker_broken_manifest(clear_rows, initialized_db):
# Delete existing tag manifest so we can reuse the tag.
TagManifestLabel.delete().execute()
TagManifest.delete().execute()
# Add a broken manifest.
broken_manifest = TagManifest.create(json_data='wat?', digest='sha256:foobar',
tag=RepositoryTag.get())
# Ensure the backfill works.
assert _backfill_manifest(broken_manifest)
# Ensure the mapping is marked as broken.
map_row = TagManifestToManifest.get(tag_manifest=broken_manifest)
assert map_row.broken
manifest_row = map_row.manifest
assert manifest_row.manifest_bytes == broken_manifest.json_data
assert manifest_row.digest == broken_manifest.digest
assert manifest_row.repository == broken_manifest.tag.repository
legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image
assert broken_manifest.tag.image == legacy_image
def test_list_alive_tags(initialized_db):
found = False
for tag in filter_to_visible_tags(filter_to_alive_tags(Tag.select())):
tags = list_alive_tags(tag.repository)
assert tag in tags
with assert_query_count(1):
legacy_images = get_legacy_images_for_tags(tags)
for tag in tags:
assert ManifestLegacyImage.get(manifest=tag.manifest).image == legacy_images[tag.id]
found = True
assert found
# Ensure hidden tags cannot be listed.
tag = Tag.get()
tag.hidden = True
tag.save()
tags = list_alive_tags(tag.repository)
assert tag not in tags
def test_tagbackfillworker(clear_all_rows, initialized_db):
# Remove the new-style rows so we can backfill.
TagToRepositoryTag.delete().execute()
Tag.delete().execute()
if clear_all_rows:
TagManifestLabelMap.delete().execute()
ManifestLabel.delete().execute()
ManifestBlob.delete().execute()
ManifestLegacyImage.delete().execute()
TagManifestToManifest.delete().execute()
Manifest.delete().execute()
found_dead_tag = False
for repository_tag in list(RepositoryTag.select()):
# Backfill the tag.
assert backfill_tag(repository_tag)
# Ensure if we try again, the backfill is skipped.
assert not backfill_tag(repository_tag)
# Ensure that we now have the expected tag rows.
tag_to_repo_tag = TagToRepositoryTag.get(repository_tag=repository_tag)
tag = tag_to_repo_tag.tag
assert tag.name == repository_tag.name
assert tag.repository == repository_tag.repository
assert not tag.hidden
assert tag.reversion == repository_tag.reversion
if repository_tag.lifetime_start_ts is None:
assert tag.lifetime_start_ms is None
else:
assert tag.lifetime_start_ms == (repository_tag.lifetime_start_ts *
1000)
if repository_tag.lifetime_end_ts is None:
assert tag.lifetime_end_ms is None
else:
assert tag.lifetime_end_ms == (repository_tag.lifetime_end_ts *
1000)
found_dead_tag = True
assert tag.manifest
# Ensure that we now have the expected manifest rows.
try:
tag_manifest = TagManifest.get(tag=repository_tag)
except TagManifest.DoesNotExist:
continue
map_row = TagManifestToManifest.get(tag_manifest=tag_manifest)
assert not map_row.broken
manifest_row = map_row.manifest
assert manifest_row.manifest_bytes == tag_manifest.json_data
assert manifest_row.digest == tag_manifest.digest
assert manifest_row.repository == tag_manifest.tag.repository
assert tag.manifest == map_row.manifest
legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image
assert tag_manifest.tag.image == legacy_image
expected_storages = {tag_manifest.tag.image.storage.id}
for parent_image_id in tag_manifest.tag.image.ancestor_id_list():
expected_storages.add(Image.get(id=parent_image_id).storage_id)
found_storages = {
manifest_blob.blob_id
for manifest_blob in ManifestBlob.select().where(
ManifestBlob.manifest == manifest_row)
}
assert expected_storages == found_storages
# Ensure the labels were copied over.
tmls = list(TagManifestLabel.select().where(
TagManifestLabel.annotated == tag_manifest))
expected_labels = {tml.label_id for tml in tmls}
found_labels = {
m.label_id
for m in ManifestLabel.select().where(
ManifestLabel.manifest == manifest_row)
}
assert found_labels == expected_labels
# Verify at the repository level.
for repository in list(Repository.select()):
tags = RepositoryTag.select().where(
RepositoryTag.repository == repository,
RepositoryTag.hidden == False)
oci_tags = Tag.select().where(Tag.repository == repository)
assert len(tags) == len(oci_tags)
assert {t.name for t in tags} == {t.name for t in oci_tags}
for tag in tags:
tag_manifest = TagManifest.get(tag=tag)
ttr = TagToRepositoryTag.get(repository_tag=tag)
manifest = ttr.tag.manifest
assert tag_manifest.json_data == manifest.manifest_bytes
assert tag_manifest.digest == manifest.digest
assert tag.image == ManifestLegacyImage.get(
manifest=manifest).image
assert tag.lifetime_start_ts == (ttr.tag.lifetime_start_ms / 1000)
if tag.lifetime_end_ts:
assert tag.lifetime_end_ts == (ttr.tag.lifetime_end_ms / 1000)
else:
assert ttr.tag.lifetime_end_ms is None
assert found_dead_tag
def _garbage_collect_manifest(manifest_id, context):
assert manifest_id is not None
# Make sure the manifest isn't referenced.
if _check_manifest_used(manifest_id):
return False
# Add the manifest's blobs to the context to be GCed.
for manifest_blob in ManifestBlob.select().where(ManifestBlob.manifest == manifest_id):
context.add_blob_id(manifest_blob.blob_id)
# Retrieve the manifest's associated image, if any.
try:
legacy_image_id = ManifestLegacyImage.get(manifest=manifest_id).image_id
context.add_legacy_image_id(legacy_image_id)
except ManifestLegacyImage.DoesNotExist:
legacy_image_id = None
# Add child manifests to be GCed.
for connector in ManifestChild.select().where(ManifestChild.manifest == manifest_id):
context.add_manifest_id(connector.child_manifest_id)
# Add the labels to be GCed.
for manifest_label in ManifestLabel.select().where(ManifestLabel.manifest == manifest_id):
context.add_label_id(manifest_label.label_id)
# Delete the manifest.
with db_transaction():
try:
manifest = Manifest.select().where(Manifest.id == manifest_id).get()
except Manifest.DoesNotExist:
return False
assert manifest.id == manifest_id
assert manifest.repository_id == context.repository.id
if _check_manifest_used(manifest_id):
return False
# Delete any label mappings.
(TagManifestLabelMap
.delete()
.where(TagManifestLabelMap.manifest == manifest_id)
.execute())
# Delete any mapping rows for the manifest.
TagManifestToManifest.delete().where(TagManifestToManifest.manifest == manifest_id).execute()
# Delete any label rows.
ManifestLabel.delete().where(ManifestLabel.manifest == manifest_id,
ManifestLabel.repository == context.repository).execute()
# Delete any child manifest rows.
ManifestChild.delete().where(ManifestChild.manifest == manifest_id,
ManifestChild.repository == context.repository).execute()
# Delete the manifest blobs for the manifest.
ManifestBlob.delete().where(ManifestBlob.manifest == manifest_id,
ManifestBlob.repository == context.repository).execute()
# Delete the manifest legacy image row.
if legacy_image_id:
(ManifestLegacyImage
.delete()
.where(ManifestLegacyImage.manifest == manifest_id,
ManifestLegacyImage.repository == context.repository)
.execute())
# Delete the manifest.
manifest.delete_instance()
context.mark_manifest_removed(manifest)
return True
def _get_legacy_image(namespace, repo, tag, include_storage=True):
repo_ref = registry_model.lookup_repository(namespace, repo)
repo_tag = registry_model.get_repo_tag(repo_ref, tag)
manifest = registry_model.get_manifest_for_tag(repo_tag)
return ManifestLegacyImage.get(manifest_id=manifest._db_id).image
def _garbage_collect_manifest(manifest_id, context):
assert manifest_id is not None
# Make sure the manifest isn't referenced.
if _check_manifest_used(manifest_id):
return False
# Add the manifest's blobs to the context to be GCed.
for manifest_blob in ManifestBlob.select().where(
ManifestBlob.manifest == manifest_id):
context.add_blob_id(manifest_blob.blob_id)
# Retrieve the manifest's associated image, if any.
try:
legacy_image_id = ManifestLegacyImage.get(
manifest=manifest_id).image_id
context.add_legacy_image_id(legacy_image_id)
except ManifestLegacyImage.DoesNotExist:
legacy_image_id = None
# Add child manifests to be GCed.
for connector in ManifestChild.select().where(
ManifestChild.manifest == manifest_id):
context.add_manifest_id(connector.child_manifest_id)
# Add the labels to be GCed.
for manifest_label in ManifestLabel.select().where(
ManifestLabel.manifest == manifest_id):
context.add_label_id(manifest_label.label_id)
# Delete the manifest.
with db_transaction():
try:
manifest = Manifest.select().where(
Manifest.id == manifest_id).get()
except Manifest.DoesNotExist:
return False
assert manifest.id == manifest_id
assert manifest.repository_id == context.repository.id
if _check_manifest_used(manifest_id):
return False
# Delete any label mappings.
deleted_tag_manifest_label_map = (TagManifestLabelMap.delete().where(
TagManifestLabelMap.manifest == manifest_id).execute())
# Delete any mapping rows for the manifest.
deleted_tag_manifest_to_manifest = (
TagManifestToManifest.delete().where(
TagManifestToManifest.manifest == manifest_id).execute())
# Delete any label rows.
deleted_manifest_label = (ManifestLabel.delete().where(
ManifestLabel.manifest == manifest_id,
ManifestLabel.repository == context.repository,
).execute())
# Delete any child manifest rows.
deleted_manifest_child = (ManifestChild.delete().where(
ManifestChild.manifest == manifest_id,
ManifestChild.repository == context.repository,
).execute())
# Delete the manifest blobs for the manifest.
deleted_manifest_blob = (ManifestBlob.delete().where(
ManifestBlob.manifest == manifest_id,
ManifestBlob.repository == context.repository).execute())
# Delete the security status for the manifest
deleted_manifest_security = (ManifestSecurityStatus.delete().where(
ManifestSecurityStatus.manifest == manifest_id,
ManifestSecurityStatus.repository == context.repository,
).execute())
# Delete the manifest legacy image row.
deleted_manifest_legacy_image = 0
if legacy_image_id:
deleted_manifest_legacy_image = (
ManifestLegacyImage.delete().where(
ManifestLegacyImage.manifest == manifest_id,
ManifestLegacyImage.repository == context.repository,
).execute())
# Delete the manifest.
manifest.delete_instance()
context.mark_manifest_removed(manifest)
gc_table_rows_deleted.labels(
table="TagManifestLabelMap").inc(deleted_tag_manifest_label_map)
gc_table_rows_deleted.labels(
table="TagManifestToManifest").inc(deleted_tag_manifest_to_manifest)
gc_table_rows_deleted.labels(
table="ManifestLabel").inc(deleted_manifest_label)
gc_table_rows_deleted.labels(
table="ManifestChild").inc(deleted_manifest_child)
gc_table_rows_deleted.labels(
table="ManifestBlob").inc(deleted_manifest_blob)
gc_table_rows_deleted.labels(
table="ManifestSecurityStatus").inc(deleted_manifest_security)
if deleted_manifest_legacy_image:
gc_table_rows_deleted.labels(
table="ManifestLegacyImage").inc(deleted_manifest_legacy_image)
gc_table_rows_deleted.labels(table="Manifest").inc()
return True
def _create_manifest(
repository_id,
manifest_interface_instance,
storage,
temp_tag_expiration_sec=TEMP_TAG_EXPIRATION_SEC,
for_tagging=False,
raise_on_error=False,
retriever=None,
):
# Validate the manifest.
retriever = retriever or RepositoryContentRetriever.for_repository(
repository_id, storage)
try:
manifest_interface_instance.validate(retriever)
except (ManifestException, MalformedSchema2ManifestList, BlobDoesNotExist,
IOError) as ex:
logger.exception("Could not validate manifest `%s`",
manifest_interface_instance.digest)
if raise_on_error:
raise CreateManifestException(str(ex))
return None
# Load, parse and get/create the child manifests, if any.
child_manifest_refs = manifest_interface_instance.child_manifests(
retriever)
child_manifest_rows = {}
child_manifest_label_dicts = []
if child_manifest_refs is not None:
for child_manifest_ref in child_manifest_refs:
# Load and parse the child manifest.
try:
child_manifest = child_manifest_ref.manifest_obj
except (
ManifestException,
MalformedSchema2ManifestList,
BlobDoesNotExist,
IOError,
) as ex:
logger.exception(
"Could not load manifest list for manifest `%s`",
manifest_interface_instance.digest,
)
if raise_on_error:
raise CreateManifestException(str(ex))
return None
# Retrieve its labels.
labels = child_manifest.get_manifest_labels(retriever)
if labels is None:
if raise_on_error:
raise CreateManifestException(
"Unable to retrieve manifest labels")
logger.exception(
"Could not load manifest labels for child manifest")
return None
# Get/create the child manifest in the database.
child_manifest_info = get_or_create_manifest(
repository_id,
child_manifest,
storage,
raise_on_error=raise_on_error)
if child_manifest_info is None:
if raise_on_error:
raise CreateManifestException(
"Unable to retrieve child manifest")
logger.error("Could not get/create child manifest")
return None
child_manifest_rows[child_manifest_info.manifest.
digest] = child_manifest_info.manifest
child_manifest_label_dicts.append(labels)
# Ensure all the blobs in the manifest exist.
digests = set(manifest_interface_instance.local_blob_digests)
blob_map = {}
# If the special empty layer is required, simply load it directly. This is much faster
# than trying to load it on a per repository basis, and that is unnecessary anyway since
# this layer is predefined.
if EMPTY_LAYER_BLOB_DIGEST in digests:
digests.remove(EMPTY_LAYER_BLOB_DIGEST)
blob_map[EMPTY_LAYER_BLOB_DIGEST] = get_shared_blob(
EMPTY_LAYER_BLOB_DIGEST)
if not blob_map[EMPTY_LAYER_BLOB_DIGEST]:
if raise_on_error:
raise CreateManifestException(
"Unable to retrieve specialized empty blob")
logger.warning("Could not find the special empty blob in storage")
return None
if digests:
query = lookup_repo_storages_by_content_checksum(
repository_id, digests)
blob_map.update({s.content_checksum: s for s in query})
for digest_str in digests:
if digest_str not in blob_map:
logger.warning(
"Unknown blob `%s` under manifest `%s` for repository `%s`",
digest_str,
manifest_interface_instance.digest,
repository_id,
)
if raise_on_error:
raise CreateManifestException("Unknown blob `%s`" %
digest_str)
return None
# Special check: If the empty layer blob is needed for this manifest, add it to the
# blob map. This is necessary because Docker decided to elide sending of this special
# empty layer in schema version 2, but we need to have it referenced for GC and schema version 1.
if EMPTY_LAYER_BLOB_DIGEST not in blob_map:
try:
requires_empty_layer = manifest_interface_instance.get_requires_empty_layer_blob(
retriever)
except ManifestException as ex:
if raise_on_error:
raise CreateManifestException(str(ex))
return None
if requires_empty_layer is None:
if raise_on_error:
raise CreateManifestException(
"Could not load configuration blob")
return None
if requires_empty_layer:
shared_blob = get_or_create_shared_blob(EMPTY_LAYER_BLOB_DIGEST,
EMPTY_LAYER_BYTES, storage)
assert not shared_blob.uploading
assert shared_blob.content_checksum == EMPTY_LAYER_BLOB_DIGEST
blob_map[EMPTY_LAYER_BLOB_DIGEST] = shared_blob
# Determine and populate the legacy image if necessary. Manifest lists will not have a legacy
# image.
legacy_image = None
if manifest_interface_instance.has_legacy_image:
legacy_image_id = _populate_legacy_image(repository_id,
manifest_interface_instance,
blob_map, retriever,
raise_on_error)
if legacy_image_id is None:
return None
legacy_image = get_image(repository_id, legacy_image_id)
if legacy_image is None:
return None
# Create the manifest and its blobs.
media_type = Manifest.media_type.get_id(
manifest_interface_instance.media_type)
storage_ids = {storage.id for storage in blob_map.values()}
with db_transaction():
# Check for the manifest. This is necessary because Postgres doesn't handle IntegrityErrors
# well under transactions.
try:
manifest = Manifest.get(repository=repository_id,
digest=manifest_interface_instance.digest)
return CreatedManifest(manifest=manifest,
newly_created=False,
labels_to_apply=None)
except Manifest.DoesNotExist:
pass
# Create the manifest.
try:
manifest = Manifest.create(
repository=repository_id,
digest=manifest_interface_instance.digest,
media_type=media_type,
manifest_bytes=manifest_interface_instance.bytes.
as_encoded_str(),
)
except IntegrityError as ie:
try:
manifest = Manifest.get(
repository=repository_id,
digest=manifest_interface_instance.digest)
except Manifest.DoesNotExist:
logger.error(
"Got integrity error when trying to create manifest: %s",
ie)
if raise_on_error:
raise CreateManifestException(
"Attempt to create an invalid manifest. Please report this issue."
)
return None
return CreatedManifest(manifest=manifest,
newly_created=False,
labels_to_apply=None)
# Insert the blobs.
blobs_to_insert = [
dict(manifest=manifest, repository=repository_id, blob=storage_id)
for storage_id in storage_ids
]
if blobs_to_insert:
ManifestBlob.insert_many(blobs_to_insert).execute()
# Set the legacy image (if applicable).
if legacy_image is not None:
ManifestLegacyImage.create(repository=repository_id,
image=legacy_image,
manifest=manifest)
# Insert the manifest child rows (if applicable).
if child_manifest_rows:
children_to_insert = [
dict(manifest=manifest,
child_manifest=child_manifest,
repository=repository_id)
for child_manifest in child_manifest_rows.values()
]
ManifestChild.insert_many(children_to_insert).execute()
# If this manifest is being created not for immediate tagging, add a temporary tag to the
# manifest to ensure it isn't being GCed. If the manifest *is* for tagging, then since we're
# creating a new one here, it cannot be GCed (since it isn't referenced by anything yet), so
# its safe to elide the temp tag operation. If we ever change GC code to collect *all* manifests
# in a repository for GC, then we will have to reevaluate this optimization at that time.
if not for_tagging:
create_temporary_tag_if_necessary(manifest,
temp_tag_expiration_sec)
# Define the labels for the manifest (if any).
# TODO: Once the old data model is gone, turn this into a batch operation and make the label
# application to the manifest occur under the transaction.
labels = manifest_interface_instance.get_manifest_labels(retriever)
if labels:
for key, value in labels.iteritems():
# NOTE: There can technically be empty label keys via Dockerfile's. We ignore any
# such `labels`, as they don't really mean anything.
if not key:
continue
media_type = "application/json" if is_json(value) else "text/plain"
create_manifest_label(manifest, key, value, "manifest", media_type)
# Return the dictionary of labels to apply (i.e. those labels that cause an action to be taken
# on the manifest or its resulting tags). We only return those labels either defined on
# the manifest or shared amongst all the child manifests. We intersect amongst all child manifests
# to ensure that any action performed is defined in all manifests.
labels_to_apply = labels or {}
if child_manifest_label_dicts:
labels_to_apply = child_manifest_label_dicts[0].viewitems()
for child_manifest_label_dict in child_manifest_label_dicts[1:]:
# Intersect the key+values of the labels to ensure we get the exact same result
# for all the child manifests.
labels_to_apply = labels_to_apply & child_manifest_label_dict.viewitems(
)
labels_to_apply = dict(labels_to_apply)
return CreatedManifest(manifest=manifest,
newly_created=True,
labels_to_apply=labels_to_apply)
