def set_tag_end_ms(tag, end_ms):
"""
Sets the end timestamp for a tag.
Should only be called by change_tag_expiration or tests.
"""
with db_transaction():
updated = (Tag.update(lifetime_end_ms=end_ms).where(
Tag.id == tag).where(
Tag.lifetime_end_ms == tag.lifetime_end_ms).execute())
if updated != 1:
return (None, False)
# TODO: Remove the linkage code once RepositoryTag is gone.
try:
old_style_tag = (TagToRepositoryTag.select(
TagToRepositoryTag, RepositoryTag).join(RepositoryTag).where(
TagToRepositoryTag.tag == tag).get()).repository_tag
old_style_tag.lifetime_end_ts = end_ms // 1000 if end_ms is not None else None
old_style_tag.save()
except TagToRepositoryTag.DoesNotExist:
pass
return (tag.lifetime_end_ms, True)
def _delete_tag(tag, now_ms):
"""
Deletes the given tag by marking it as expired.
"""
now_ts = int(now_ms // 1000)
with db_transaction():
updated = (Tag.update(lifetime_end_ms=now_ms).where(
Tag.id == tag.id,
Tag.lifetime_end_ms == tag.lifetime_end_ms).execute())
if updated != 1:
return None
# TODO: Remove the linkage code once RepositoryTag is gone.
try:
old_style_tag = (TagToRepositoryTag.select(
TagToRepositoryTag, RepositoryTag).join(RepositoryTag).where(
TagToRepositoryTag.tag == tag).get()).repository_tag
old_style_tag.lifetime_end_ts = now_ts
old_style_tag.save()
except TagToRepositoryTag.DoesNotExist:
pass
return tag
def upgrade_progress():
total_tags = RepositoryTag.select().where(RepositoryTag.hidden == False).count()
if total_tags == 0:
return jsonify({
'progress': 1.0,
'tags_remaining': 0,
'total_tags': 0,
})
upgraded_tags = TagToRepositoryTag.select().count()
return jsonify({
'progress': float(upgraded_tags) / total_tags,
'tags_remaining': total_tags - upgraded_tags,
'total_tags': total_tags,
})
def delete_tag(namespace_name, repository_name, tag_name, now_ms=None):
now_ms = now_ms or get_epoch_timestamp_ms()
now_ts = int(now_ms / 1000)
with db_transaction():
try:
query = _tag_alive(
RepositoryTag.select(
RepositoryTag, Repository).join(Repository).join(
Namespace,
on=(Repository.namespace_user == Namespace.id)).where(
Repository.name == repository_name,
Namespace.username == namespace_name,
RepositoryTag.name == tag_name,
),
now_ts,
)
found = db_for_update(query).get()
except RepositoryTag.DoesNotExist:
msg = "Invalid repository tag '%s' on repository '%s/%s'" % (
tag_name,
namespace_name,
repository_name,
)
raise DataModelException(msg)
found.lifetime_end_ts = now_ts
found.save()
try:
oci_tag_query = TagToRepositoryTag.select().where(
TagToRepositoryTag.repository_tag == found)
oci_tag = db_for_update(oci_tag_query).get().tag
oci_tag.lifetime_end_ms = now_ms
oci_tag.save()
except TagToRepositoryTag.DoesNotExist:
pass
return found
def assert_gc_integrity(expect_storage_removed=True, check_oci_tags=True):
""" Specialized assertion for ensuring that GC cleans up all dangling storages
and labels, invokes the callback for images removed and doesn't invoke the
callback for images *not* removed.
"""
# Add a callback for when images are removed.
removed_image_storages = []
model.config.register_image_cleanup_callback(removed_image_storages.extend)
# Store the number of dangling storages and labels.
existing_storage_count = _get_dangling_storage_count()
existing_label_count = _get_dangling_label_count()
existing_manifest_count = _get_dangling_manifest_count()
yield
# Ensure the number of dangling storages, manifests and labels has not changed.
updated_storage_count = _get_dangling_storage_count()
assert updated_storage_count == existing_storage_count
updated_label_count = _get_dangling_label_count()
assert updated_label_count == existing_label_count, _get_dangling_labels()
updated_manifest_count = _get_dangling_manifest_count()
assert updated_manifest_count == existing_manifest_count
# Ensure that for each call to the image+storage cleanup callback, the image and its
# storage is not found *anywhere* in the database.
for removed_image_and_storage in removed_image_storages:
with pytest.raises(Image.DoesNotExist):
Image.get(id=removed_image_and_storage.id)
# Ensure that image storages are only removed if not shared.
shared = Image.select().where(
Image.storage == removed_image_and_storage.storage_id).count()
if shared == 0:
shared = (ManifestBlob.select().where(
ManifestBlob.blob ==
removed_image_and_storage.storage_id).count())
if shared == 0:
with pytest.raises(ImageStorage.DoesNotExist):
ImageStorage.get(id=removed_image_and_storage.storage_id)
with pytest.raises(ImageStorage.DoesNotExist):
ImageStorage.get(uuid=removed_image_and_storage.storage.uuid)
# Ensure all CAS storage is in the storage engine.
preferred = storage.preferred_locations[0]
for storage_row in ImageStorage.select():
if storage_row.cas_path:
storage.get_content({preferred},
storage.blob_path(
storage_row.content_checksum))
for blob_row in ApprBlob.select():
storage.get_content({preferred}, storage.blob_path(blob_row.digest))
# Ensure there are no danglings OCI tags.
if check_oci_tags:
oci_tags = {t.id for t in Tag.select()}
referenced_oci_tags = {t.tag_id for t in TagToRepositoryTag.select()}
assert not oci_tags - referenced_oci_tags
# Ensure all tags have valid manifests.
for manifest in {t.manifest for t in Tag.select()}:
# Ensure that the manifest's blobs all exist.
found_blobs = {
b.blob.content_checksum
for b in ManifestBlob.select().where(
ManifestBlob.manifest == manifest)
}
parsed = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(manifest.manifest_bytes),
manifest.media_type.name)
assert set(parsed.local_blob_digests) == found_blobs