def _backfill_labels(tag_manifest, manifest, repository):
tmls = list(TagManifestLabel.select().where(
TagManifestLabel.annotated == tag_manifest))
if not tmls:
return
for tag_manifest_label in tmls:
label = tag_manifest_label.label
try:
TagManifestLabelMap.get(tag_manifest_label=tag_manifest_label)
continue
except TagManifestLabelMap.DoesNotExist:
pass
try:
manifest_label = ManifestLabel.create(manifest=manifest,
label=label,
repository=repository)
TagManifestLabelMap.create(
manifest_label=manifest_label,
tag_manifest_label=tag_manifest_label,
label=label,
manifest=manifest,
tag_manifest=tag_manifest_label.annotated)
except IntegrityError:
continue
def create_manifest_label(tag_manifest,
key,
value,
source_type_name,
media_type_name=None):
"""
Creates a new manifest label on a specific tag manifest.
"""
if not key:
raise InvalidLabelKeyException("Missing key on label")
# Note that we don't prevent invalid label names coming from the manifest to be stored, as Docker
# does not currently prevent them from being put into said manifests.
if not validate_label_key(key) and source_type_name != "manifest":
raise InvalidLabelKeyException(
"Label key `%s` is invalid or reserved" % key)
# Find the matching media type. If none specified, we infer.
if media_type_name is None:
media_type_name = "text/plain"
if is_json(value):
media_type_name = "application/json"
media_type_id = _get_media_type_id(media_type_name)
if media_type_id is None:
raise InvalidMediaTypeException()
source_type_id = _get_label_source_type_id(source_type_name)
with db_transaction():
label = Label.create(key=key,
value=value,
source_type=source_type_id,
media_type=media_type_id)
tag_manifest_label = TagManifestLabel.create(
annotated=tag_manifest,
label=label,
repository=tag_manifest.tag.repository)
try:
mapping_row = TagManifestToManifest.get(tag_manifest=tag_manifest)
if mapping_row.manifest:
manifest_label = ManifestLabel.create(
manifest=mapping_row.manifest,
label=label,
repository=tag_manifest.tag.repository,
)
TagManifestLabelMap.create(
manifest_label=manifest_label,
tag_manifest_label=tag_manifest_label,
label=label,
manifest=mapping_row.manifest,
tag_manifest=tag_manifest,
)
except TagManifestToManifest.DoesNotExist:
pass
return label
def clear_rows(initialized_db):
# Remove all new-style rows so we can backfill.
TagToRepositoryTag.delete().execute()
Tag.delete().execute()
TagManifestLabelMap.delete().execute()
ManifestLabel.delete().execute()
ManifestBlob.delete().execute()
ManifestLegacyImage.delete().execute()
TagManifestToManifest.delete().execute()
Manifest.delete().execute()
def delete_manifest_label(label_uuid, manifest):
""" Deletes the manifest label on the tag manifest with the given ID. Returns the label deleted
or None if none.
"""
# Find the label itself.
label = get_manifest_label(label_uuid, manifest)
if label is None:
return None
if not label.source_type.mutable:
raise DataModelException("Cannot delete immutable label")
# Delete the mapping records and label.
# TODO: Remove this code once the TagManifest table is gone.
with db_transaction():
(TagManifestLabelMap.delete().where(
TagManifestLabelMap.label == label).execute())
deleted_count = TagManifestLabel.delete().where(
TagManifestLabel.label == label).execute()
if deleted_count != 1:
logger.warning(
"More than a single label deleted for matching label %s",
label_uuid)
deleted_count = ManifestLabel.delete().where(
ManifestLabel.label == label).execute()
if deleted_count != 1:
logger.warning(
"More than a single label deleted for matching label %s",
label_uuid)
label.delete_instance(recursive=False)
return label
def delete_manifest_label(label_uuid, tag_manifest):
""" Deletes the manifest label on the tag manifest with the given ID. """
# Find the label itself.
label = get_manifest_label(label_uuid, tag_manifest)
if label is None:
return None
if not label.source_type.mutable:
raise DataModelException('Cannot delete immutable label')
# Delete the mapping records and label.
(TagManifestLabelMap.delete().where(
TagManifestLabelMap.label == label).execute())
deleted_count = TagManifestLabel.delete().where(
TagManifestLabel.label == label).execute()
if deleted_count != 1:
logger.warning(
'More than a single label deleted for matching label %s',
label_uuid)
deleted_count = ManifestLabel.delete().where(
ManifestLabel.label == label).execute()
if deleted_count != 1:
logger.warning(
'More than a single label deleted for matching label %s',
label_uuid)
label.delete_instance(recursive=False)
return label
def backfill_label(tag_manifest_label):
logger.info("Backfilling label %s", tag_manifest_label.id)
# Ensure that a mapping row doesn't already exist. If it does, we've been preempted.
if lookup_map_row(tag_manifest_label):
return False
# Ensure the tag manifest has been backfilled into the manifest table.
try:
tmt = TagManifestToManifest.get(
tag_manifest=tag_manifest_label.annotated)
except TagManifestToManifest.DoesNotExist:
# We'll come back to this later.
logger.debug(
"Tag Manifest %s for label %s has not yet been backfilled",
tag_manifest_label.annotated.id,
tag_manifest_label.id,
)
return True
repository = tag_manifest_label.repository
# Create the new mapping entry and label.
with db_transaction():
if lookup_map_row(tag_manifest_label):
return False
label = tag_manifest_label.label
if tmt.manifest:
try:
manifest_label = ManifestLabel.create(manifest=tmt.manifest,
label=label,
repository=repository)
TagManifestLabelMap.create(
manifest_label=manifest_label,
tag_manifest_label=tag_manifest_label,
label=label,
manifest=tmt.manifest,
tag_manifest=tag_manifest_label.annotated,
)
except IntegrityError:
return False
logger.info("Backfilled label %s", tag_manifest_label.id)
return True
def _garbage_collect_legacy_manifest(legacy_manifest_id, context):
assert legacy_manifest_id is not None
# Add the labels to be GCed.
query = TagManifestLabel.select().where(
TagManifestLabel.annotated == legacy_manifest_id)
for manifest_label in query:
context.add_label_id(manifest_label.label_id)
# Delete the tag manifest.
with db_transaction():
try:
tag_manifest = TagManifest.select().where(
TagManifest.id == legacy_manifest_id).get()
except TagManifest.DoesNotExist:
return False
assert tag_manifest.id == legacy_manifest_id
assert tag_manifest.tag.repository_id == context.repository.id
# Delete any label mapping rows.
(TagManifestLabelMap.delete().where(
TagManifestLabelMap.tag_manifest == legacy_manifest_id).execute())
# Delete the label rows.
TagManifestLabel.delete().where(
TagManifestLabel.annotated == legacy_manifest_id).execute()
# Delete the mapping row if it exists.
try:
tmt = (TagManifestToManifest.select().where(
TagManifestToManifest.tag_manifest == tag_manifest).get())
context.add_manifest_id(tmt.manifest_id)
tmt_deleted = tmt.delete_instance()
if tmt_deleted:
gc_table_rows_deleted.labels(
table="TagManifestToManifest").inc()
except TagManifestToManifest.DoesNotExist:
pass
# Delete the tag manifest.
tag_manifest_deleted = tag_manifest.delete_instance()
if tag_manifest_deleted:
gc_table_rows_deleted.labels(table="TagManifest").inc()
return True
def lookup_map_row(tag_manifest_label):
try:
TagManifestLabelMap.get(tag_manifest_label=tag_manifest_label)
return True
except TagManifestLabelMap.DoesNotExist:
return False
def _garbage_collect_manifest(manifest_id, context):
assert manifest_id is not None
# Make sure the manifest isn't referenced.
if _check_manifest_used(manifest_id):
return False
# Add the manifest's blobs to the context to be GCed.
for manifest_blob in ManifestBlob.select().where(
ManifestBlob.manifest == manifest_id):
context.add_blob_id(manifest_blob.blob_id)
# Retrieve the manifest's associated image, if any.
try:
legacy_image_id = ManifestLegacyImage.get(
manifest=manifest_id).image_id
context.add_legacy_image_id(legacy_image_id)
except ManifestLegacyImage.DoesNotExist:
legacy_image_id = None
# Add child manifests to be GCed.
for connector in ManifestChild.select().where(
ManifestChild.manifest == manifest_id):
context.add_manifest_id(connector.child_manifest_id)
# Add the labels to be GCed.
for manifest_label in ManifestLabel.select().where(
ManifestLabel.manifest == manifest_id):
context.add_label_id(manifest_label.label_id)
# Delete the manifest.
with db_transaction():
try:
manifest = Manifest.select().where(
Manifest.id == manifest_id).get()
except Manifest.DoesNotExist:
return False
assert manifest.id == manifest_id
assert manifest.repository_id == context.repository.id
if _check_manifest_used(manifest_id):
return False
# Delete any label mappings.
deleted_tag_manifest_label_map = (TagManifestLabelMap.delete().where(
TagManifestLabelMap.manifest == manifest_id).execute())
# Delete any mapping rows for the manifest.
deleted_tag_manifest_to_manifest = (
TagManifestToManifest.delete().where(
TagManifestToManifest.manifest == manifest_id).execute())
# Delete any label rows.
deleted_manifest_label = (ManifestLabel.delete().where(
ManifestLabel.manifest == manifest_id,
ManifestLabel.repository == context.repository,
).execute())
# Delete any child manifest rows.
deleted_manifest_child = (ManifestChild.delete().where(
ManifestChild.manifest == manifest_id,
ManifestChild.repository == context.repository,
).execute())
# Delete the manifest blobs for the manifest.
deleted_manifest_blob = (ManifestBlob.delete().where(
ManifestBlob.manifest == manifest_id,
ManifestBlob.repository == context.repository).execute())
# Delete the security status for the manifest
deleted_manifest_security = (ManifestSecurityStatus.delete().where(
ManifestSecurityStatus.manifest == manifest_id,
ManifestSecurityStatus.repository == context.repository,
).execute())
# Delete the manifest legacy image row.
deleted_manifest_legacy_image = 0
if legacy_image_id:
deleted_manifest_legacy_image = (
ManifestLegacyImage.delete().where(
ManifestLegacyImage.manifest == manifest_id,
ManifestLegacyImage.repository == context.repository,
).execute())
# Delete the manifest.
manifest.delete_instance()
context.mark_manifest_removed(manifest)
gc_table_rows_deleted.labels(
table="TagManifestLabelMap").inc(deleted_tag_manifest_label_map)
gc_table_rows_deleted.labels(
table="TagManifestToManifest").inc(deleted_tag_manifest_to_manifest)
gc_table_rows_deleted.labels(
table="ManifestLabel").inc(deleted_manifest_label)
gc_table_rows_deleted.labels(
table="ManifestChild").inc(deleted_manifest_child)
gc_table_rows_deleted.labels(
table="ManifestBlob").inc(deleted_manifest_blob)
gc_table_rows_deleted.labels(
table="ManifestSecurityStatus").inc(deleted_manifest_security)
if deleted_manifest_legacy_image:
gc_table_rows_deleted.labels(
table="ManifestLegacyImage").inc(deleted_manifest_legacy_image)
gc_table_rows_deleted.labels(table="Manifest").inc()
return True
def test_tagbackfillworker(clear_all_rows, initialized_db):
# Remove the new-style rows so we can backfill.
TagToRepositoryTag.delete().execute()
Tag.delete().execute()
if clear_all_rows:
TagManifestLabelMap.delete().execute()
ManifestLabel.delete().execute()
ManifestBlob.delete().execute()
ManifestLegacyImage.delete().execute()
TagManifestToManifest.delete().execute()
Manifest.delete().execute()
found_dead_tag = False
for repository_tag in list(RepositoryTag.select()):
# Backfill the tag.
assert backfill_tag(repository_tag)
# Ensure if we try again, the backfill is skipped.
assert not backfill_tag(repository_tag)
# Ensure that we now have the expected tag rows.
tag_to_repo_tag = TagToRepositoryTag.get(repository_tag=repository_tag)
tag = tag_to_repo_tag.tag
assert tag.name == repository_tag.name
assert tag.repository == repository_tag.repository
assert not tag.hidden
assert tag.reversion == repository_tag.reversion
if repository_tag.lifetime_start_ts is None:
assert tag.lifetime_start_ms is None
else:
assert tag.lifetime_start_ms == (repository_tag.lifetime_start_ts *
1000)
if repository_tag.lifetime_end_ts is None:
assert tag.lifetime_end_ms is None
else:
assert tag.lifetime_end_ms == (repository_tag.lifetime_end_ts *
1000)
found_dead_tag = True
assert tag.manifest
# Ensure that we now have the expected manifest rows.
try:
tag_manifest = TagManifest.get(tag=repository_tag)
except TagManifest.DoesNotExist:
continue
map_row = TagManifestToManifest.get(tag_manifest=tag_manifest)
assert not map_row.broken
manifest_row = map_row.manifest
assert manifest_row.manifest_bytes == tag_manifest.json_data
assert manifest_row.digest == tag_manifest.digest
assert manifest_row.repository == tag_manifest.tag.repository
assert tag.manifest == map_row.manifest
legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image
assert tag_manifest.tag.image == legacy_image
expected_storages = {tag_manifest.tag.image.storage.id}
for parent_image_id in tag_manifest.tag.image.ancestor_id_list():
expected_storages.add(Image.get(id=parent_image_id).storage_id)
found_storages = {
manifest_blob.blob_id
for manifest_blob in ManifestBlob.select().where(
ManifestBlob.manifest == manifest_row)
}
assert expected_storages == found_storages
# Ensure the labels were copied over.
tmls = list(TagManifestLabel.select().where(
TagManifestLabel.annotated == tag_manifest))
expected_labels = {tml.label_id for tml in tmls}
found_labels = {
m.label_id
for m in ManifestLabel.select().where(
ManifestLabel.manifest == manifest_row)
}
assert found_labels == expected_labels
# Verify at the repository level.
for repository in list(Repository.select()):
tags = RepositoryTag.select().where(
RepositoryTag.repository == repository,
RepositoryTag.hidden == False)
oci_tags = Tag.select().where(Tag.repository == repository)
assert len(tags) == len(oci_tags)
assert {t.name for t in tags} == {t.name for t in oci_tags}
for tag in tags:
tag_manifest = TagManifest.get(tag=tag)
ttr = TagToRepositoryTag.get(repository_tag=tag)
manifest = ttr.tag.manifest
assert tag_manifest.json_data == manifest.manifest_bytes
assert tag_manifest.digest == manifest.digest
assert tag.image == ManifestLegacyImage.get(
manifest=manifest).image
assert tag.lifetime_start_ts == (ttr.tag.lifetime_start_ms / 1000)
if tag.lifetime_end_ts:
assert tag.lifetime_end_ts == (ttr.tag.lifetime_end_ms / 1000)
else:
assert ttr.tag.lifetime_end_ms is None
assert found_dead_tag
def _garbage_collect_manifest(manifest_id, context):
assert manifest_id is not None
# Make sure the manifest isn't referenced.
if _check_manifest_used(manifest_id):
return False
# Add the manifest's blobs to the context to be GCed.
for manifest_blob in ManifestBlob.select().where(ManifestBlob.manifest == manifest_id):
context.add_blob_id(manifest_blob.blob_id)
# Retrieve the manifest's associated image, if any.
try:
legacy_image_id = ManifestLegacyImage.get(manifest=manifest_id).image_id
context.add_legacy_image_id(legacy_image_id)
except ManifestLegacyImage.DoesNotExist:
legacy_image_id = None
# Add child manifests to be GCed.
for connector in ManifestChild.select().where(ManifestChild.manifest == manifest_id):
context.add_manifest_id(connector.child_manifest_id)
# Add the labels to be GCed.
for manifest_label in ManifestLabel.select().where(ManifestLabel.manifest == manifest_id):
context.add_label_id(manifest_label.label_id)
# Delete the manifest.
with db_transaction():
try:
manifest = Manifest.select().where(Manifest.id == manifest_id).get()
except Manifest.DoesNotExist:
return False
assert manifest.id == manifest_id
assert manifest.repository_id == context.repository.id
if _check_manifest_used(manifest_id):
return False
# Delete any label mappings.
(TagManifestLabelMap.delete().where(TagManifestLabelMap.manifest == manifest_id).execute())
# Delete any mapping rows for the manifest.
TagManifestToManifest.delete().where(
TagManifestToManifest.manifest == manifest_id
).execute()
# Delete any label rows.
ManifestLabel.delete().where(
ManifestLabel.manifest == manifest_id, ManifestLabel.repository == context.repository
).execute()
# Delete any child manifest rows.
ManifestChild.delete().where(
ManifestChild.manifest == manifest_id, ManifestChild.repository == context.repository
).execute()
# Delete the manifest blobs for the manifest.
ManifestBlob.delete().where(
ManifestBlob.manifest == manifest_id, ManifestBlob.repository == context.repository
).execute()
# Delete the manifest legacy image row.
if legacy_image_id:
(
ManifestLegacyImage.delete()
.where(
ManifestLegacyImage.manifest == manifest_id,
ManifestLegacyImage.repository == context.repository,
)
.execute()
)
# Delete the manifest.
manifest.delete_instance()
context.mark_manifest_removed(manifest)
return True
def create_manifest_label(manifest_id, key, value, source_type_name, media_type_name=None,
adjust_old_model=True):
""" Creates a new manifest label on a specific tag manifest. """
if not key:
raise InvalidLabelKeyException()
# Note that we don't prevent invalid label names coming from the manifest to be stored, as Docker
# does not currently prevent them from being put into said manifests.
if not validate_label_key(key) and source_type_name != 'manifest':
raise InvalidLabelKeyException('Key `%s` is invalid' % key)
# Find the matching media type. If none specified, we infer.
if media_type_name is None:
media_type_name = 'text/plain'
if is_json(value):
media_type_name = 'application/json'
try:
media_type_id = Label.media_type.get_id(media_type_name)
except MediaType.DoesNotExist:
raise InvalidMediaTypeException()
source_type_id = Label.source_type.get_id(source_type_name)
# Ensure the manifest exists.
try:
manifest = (Manifest
.select(Manifest, Repository)
.join(Repository)
.where(Manifest.id == manifest_id)
.get())
except Manifest.DoesNotExist:
return None
repository = manifest.repository
# TODO: Remove this code once the TagManifest table is gone.
tag_manifest = None
if adjust_old_model:
try:
mapping_row = (TagManifestToManifest
.select(TagManifestToManifest, TagManifest)
.join(TagManifest)
.where(TagManifestToManifest.manifest == manifest)
.get())
tag_manifest = mapping_row.tag_manifest
except TagManifestToManifest.DoesNotExist:
tag_manifest = None
with db_transaction():
label = Label.create(key=key, value=value, source_type=source_type_id, media_type=media_type_id)
manifest_label = ManifestLabel.create(manifest=manifest_id, label=label, repository=repository)
# If there exists a mapping to a TagManifest, add the old-style label.
# TODO: Remove this code once the TagManifest table is gone.
if tag_manifest:
tag_manifest_label = TagManifestLabel.create(annotated=tag_manifest, label=label,
repository=repository)
TagManifestLabelMap.create(manifest_label=manifest_label,
tag_manifest_label=tag_manifest_label,
label=label,
manifest=manifest,
tag_manifest=tag_manifest)
return label