def find_repository_with_garbage(limit_to_gc_policy_s):
""" Returns a repository that has garbage (defined as an expired Tag that is past
the repo's namespace's expiration window) or None if none.
"""
expiration_timestamp = get_epoch_timestamp_ms() - (limit_to_gc_policy_s *
1000)
try:
candidates = (Tag.select(Tag.repository).join(Repository).join(
Namespace, on=(Repository.namespace_user == Namespace.id)).where(
~(Tag.lifetime_end_ms >> None),
(Tag.lifetime_end_ms <= expiration_timestamp),
(Namespace.removed_tag_expiration_s == limit_to_gc_policy_s),
(Namespace.enabled == True),
(Repository.state != RepositoryState.MARKED_FOR_DELETION),
).limit(GC_CANDIDATE_COUNT).distinct().alias("candidates"))
found = (Tag.select(
candidates.c.repository_id).from_(candidates).order_by(
db_random_func()).get())
if found is None:
return
return Repository.get(Repository.id == found.repository_id)
except Tag.DoesNotExist:
return None
except Repository.DoesNotExist:
return None
def get_tag_by_id(tag_id):
""" Returns the tag with the given ID, joined with its manifest or None if none. """
try:
return Tag.select(
Tag, Manifest).join(Manifest).where(Tag.id == tag_id).get()
except Tag.DoesNotExist:
return None
def create_temporary_tag_if_necessary(manifest, expiration_sec):
"""
Creates a temporary tag pointing to the given manifest, with the given expiration in seconds,
unless there is an existing tag that will keep the manifest around.
"""
tag_name = "$temp-%s" % str(uuid.uuid4())
now_ms = get_epoch_timestamp_ms()
end_ms = now_ms + (expiration_sec * 1000)
# Check if there is an existing tag on the manifest that won't expire within the
# timeframe. If so, no need for a temporary tag.
with db_transaction():
try:
(Tag.select().where(
Tag.manifest == manifest,
(Tag.lifetime_end_ms >> None) |
(Tag.lifetime_end_ms >= end_ms),
).get())
return None
except Tag.DoesNotExist:
pass
return Tag.create(
name=tag_name,
repository=manifest.repository_id,
lifetime_start_ms=now_ms,
lifetime_end_ms=end_ms,
reversion=False,
hidden=True,
manifest=manifest,
tag_kind=Tag.tag_kind.get_id("tag"),
)
def set_tag_end_ts(tag, end_ts):
""" Sets the end timestamp for a tag. Should only be called by change_tag_expiration
or tests.
"""
end_ms = end_ts * 1000 if end_ts is not None else None
with db_transaction():
# Note: We check not just the ID of the tag but also its lifetime_end_ts, to ensure that it has
# not changed while we were updating it expiration.
result = (RepositoryTag.update(lifetime_end_ts=end_ts).where(
RepositoryTag.id == tag.id,
RepositoryTag.lifetime_end_ts == tag.lifetime_end_ts).execute())
# Check for a mapping to an OCI tag.
try:
oci_tag = (Tag.select().join(TagToRepositoryTag).where(
TagToRepositoryTag.repository_tag == tag).get())
(Tag.update(lifetime_end_ms=end_ms).where(
Tag.id == oci_tag.id,
Tag.lifetime_end_ms == oci_tag.lifetime_end_ms).execute())
except Tag.DoesNotExist:
pass
return (tag.lifetime_end_ts, result > 0)
def find_matching_tag(repository_id, tag_names, tag_kinds=None):
"""
Finds an alive tag in the specified repository with one of the specified tag names and returns
it or None if none.
Tag's returned are joined with their manifest.
"""
assert repository_id
assert tag_names
query = (
Tag.select(Tag, Manifest)
.join(Manifest)
.where(Tag.repository == repository_id)
.where(Tag.name << tag_names)
)
if tag_kinds:
query = query.where(Tag.tag_kind << tag_kinds)
try:
found = filter_to_alive_tags(query).get()
assert not found.hidden
return found
except Tag.DoesNotExist:
return None
def cache_namespace_repository_sizes(namespace_name):
namespace = user.get_user_or_org(namespace_name)
now_ms = get_epoch_timestamp_ms()
subquery = (Tag.select(Tag.repository_id).where(
Tag.hidden == False).where((Tag.lifetime_end_ms >> None)
| (Tag.lifetime_end_ms > now_ms)).group_by(
Tag.repository_id).having(
fn.Count(Tag.name) > 0))
namespace_repo_sizes = (Manifest.select(
(Repository.id).alias("repository_id"),
(Repository.name).alias("repository_name"),
fn.sum(Manifest.layers_compressed_size).alias("repository_size"),
).join(Repository).join(
subquery, on=(subquery.c.repository_id == Repository.id)).where(
Repository.namespace_user == namespace.id).group_by(Repository.id))
insert_query = (namespace_repo_sizes.select(
Repository.id, fn.sum(Manifest.layers_compressed_size)).join_from(
Repository, RepositorySize,
JOIN.LEFT_OUTER).where(RepositorySize.repository_id.is_null()))
RepositorySize.insert_from(
insert_query,
fields=[RepositorySize.repository_id, RepositorySize.size_bytes],
).execute()
def test_create_temp_tags_for_newly_created_sub_manifests_on_manifest_list(
self, create_repo):
repo_ref = create_repo(self.orgname, self.upstream_repository,
self.user)
input_manifest = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(UBI8_LATEST_MANIFEST_LIST),
DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE,
sparse_manifest_support=True,
)
proxy_model = ProxyModel(
self.orgname,
self.upstream_repository,
self.user,
)
manifest, _ = proxy_model._create_manifest_and_retarget_tag(
repo_ref, input_manifest, self.tag)
mchildren = ManifestChild.select(
ManifestChild.child_manifest_id).where(
ManifestChild.manifest == manifest.id)
tags = Tag.select().join(
ManifestChild,
on=(Tag.manifest_id == ManifestChild.child_manifest_id))
assert mchildren.count() == tags.count()
assert all([t.hidden
for t in tags]), "all sub manifest tags must be hidden"
assert all(
[t.name != self.tag for t in tags]
), "sub manifest tags must have temp tag name, not parent manifest name"
def test_list_repository_tag_history_all_tags(initialized_db):
for tag in Tag.select():
repo = tag.repository
with assert_query_count(1):
results, _ = list_repository_tag_history(repo, 1, 1000)
assert (tag in results) == (not tag.hidden)
def list_repository_tag_history(repository_id,
page,
page_size,
specific_tag_name=None,
active_tags_only=False,
since_time_ms=None):
""" Returns a tuple of the full set of tags found in the specified repository, including those
that are no longer alive (unless active_tags_only is True), and whether additional tags exist.
If specific_tag_name is given, the tags are further filtered by name. If since is given, tags
are further filtered to newer than that date.
Note that the returned Manifest will not contain the manifest contents.
"""
query = (Tag.select(Tag, Manifest.id, Manifest.digest,
Manifest.media_type).join(Manifest).where(
Tag.repository == repository_id).order_by(
Tag.lifetime_start_ms.desc(),
Tag.name).limit(page_size + 1).offset(
page_size * (page - 1)))
if specific_tag_name is not None:
query = query.where(Tag.name == specific_tag_name)
if since_time_ms is not None:
query = query.where((Tag.lifetime_start_ms > since_time_ms)
| (Tag.lifetime_end_ms > since_time_ms))
if active_tags_only:
query = filter_to_alive_tags(query)
query = filter_to_visible_tags(query)
results = list(query)
return results[0:page_size], len(results) > page_size
def _purge_oci_tag(tag, context, allow_non_expired=False):
assert tag.repository_id == context.repository.id
if not allow_non_expired:
assert tag.lifetime_end_ms is not None
assert tag.lifetime_end_ms <= oci_tag.get_epoch_timestamp_ms()
# Add the manifest to be GCed.
context.add_manifest_id(tag.manifest_id)
with db_transaction():
# Reload the tag and verify its lifetime_end_ms has not changed.
try:
reloaded_tag = db_for_update(Tag.select().where(Tag.id == tag.id)).get()
except Tag.DoesNotExist:
return False
assert reloaded_tag.id == tag.id
assert reloaded_tag.repository_id == context.repository.id
if reloaded_tag.lifetime_end_ms != tag.lifetime_end_ms:
return False
# Delete mapping rows.
TagToRepositoryTag.delete().where(TagToRepositoryTag.tag == tag).execute()
# Delete the tag.
tag.delete_instance()
def list_alive_tags(repository_id):
""" Returns a list of all the tags alive in the specified repository.
Tag's returned are joined with their manifest.
"""
query = (Tag.select(
Tag, Manifest).join(Manifest).where(Tag.repository == repository_id))
return filter_to_alive_tags(query)
def test_lookup_manifest(initialized_db):
found = False
for tag in filter_to_alive_tags(Tag.select()):
found = True
repo = tag.repository
digest = tag.manifest.digest
with assert_query_count(1):
assert lookup_manifest(repo, digest) == tag.manifest
assert found
for tag in Tag.select():
repo = tag.repository
digest = tag.manifest.digest
with assert_query_count(1):
assert lookup_manifest(repo, digest,
allow_dead=True) == tag.manifest
def _purge_repository_contents(repo):
""" Purges all the contents of a repository, removing all of its tags,
manifests and images.
"""
logger.debug('Purging repository %s', repo)
# Purge via all the tags.
while True:
found = False
for tags in _chunk_iterate_for_deletion(Tag.select().where(Tag.repository == repo)):
logger.debug('Found %s tags to GC under repository %s', len(tags), repo)
found = True
context = _GarbageCollectorContext(repo)
for tag in tags:
logger.debug('Deleting tag %s under repository %s', tag, repo)
assert tag.repository_id == repo.id
_purge_oci_tag(tag, context, allow_non_expired=True)
_run_garbage_collection(context)
if not found:
break
# TODO: remove this once we're fully on the OCI data model.
while True:
found = False
repo_tag_query = RepositoryTag.select().where(RepositoryTag.repository == repo)
for tags in _chunk_iterate_for_deletion(repo_tag_query):
logger.debug('Found %s tags to GC under repository %s', len(tags), repo)
found = True
context = _GarbageCollectorContext(repo)
for tag in tags:
logger.debug('Deleting tag %s under repository %s', tag, repo)
assert tag.repository_id == repo.id
_purge_pre_oci_tag(tag, context, allow_non_expired=True)
_run_garbage_collection(context)
if not found:
break
# Add all remaining images to a new context. We do this here to minimize the number of images
# we need to load.
while True:
found_image = False
image_context = _GarbageCollectorContext(repo)
for image in Image.select().where(Image.repository == repo):
found_image = True
logger.debug('Deleting image %s under repository %s', image, repo)
assert image.repository_id == repo.id
image_context.add_legacy_image_id(image.id)
_run_garbage_collection(image_context)
if not found_image:
break
def get_tags_for_legacy_image(image_id):
""" Returns the Tag's that have the associated legacy image.
NOTE: This is for legacy support in the old security notification worker and should
be removed once that code is no longer necessary.
"""
return filter_to_alive_tags(
Tag.select().distinct().join(Manifest).join(ManifestLegacyImage).where(
ManifestLegacyImage.image == image_id))
def test_delete_tags_for_manifest(initialized_db):
for tag in list(filter_to_visible_tags(filter_to_alive_tags(Tag.select()))):
repo = tag.repository
assert get_tag(repo, tag.name) == tag
with assert_query_count(4):
assert delete_tags_for_manifest(tag.manifest) == [tag]
assert get_tag(repo, tag.name) is None
def purge_repository(repo, force=False):
"""
Completely delete all traces of the repository.
Will return True upon complete success, and False upon partial or total failure. Garbage
collection is incremental and repeatable, so this return value does not need to be checked or
responded to.
"""
assert repo.state == RepositoryState.MARKED_FOR_DELETION or force
# Delete the repository of all Appr-referenced entries.
# Note that new-model Tag's must be deleted in *two* passes, as they can reference parent tags,
# and MySQL is... particular... about such relationships when deleting.
if repo.kind.name == "application":
ApprTag.delete().where(ApprTag.repository == repo,
~(ApprTag.linked_tag >> None)).execute()
ApprTag.delete().where(ApprTag.repository == repo).execute()
else:
# GC to remove the images and storage.
_purge_repository_contents(repo)
# Ensure there are no additional tags, manifests, images or blobs in the repository.
assert ApprTag.select().where(ApprTag.repository == repo).count() == 0
assert Tag.select().where(Tag.repository == repo).count() == 0
assert RepositoryTag.select().where(
RepositoryTag.repository == repo).count() == 0
assert Manifest.select().where(Manifest.repository == repo).count() == 0
assert ManifestBlob.select().where(
ManifestBlob.repository == repo).count() == 0
assert Image.select().where(Image.repository == repo).count() == 0
# Delete any repository build triggers, builds, and any other large-ish reference tables for
# the repository.
_chunk_delete_all(repo, RepositoryPermission, force=force)
_chunk_delete_all(repo, RepositoryBuild, force=force)
_chunk_delete_all(repo, RepositoryBuildTrigger, force=force)
_chunk_delete_all(repo, RepositoryActionCount, force=force)
_chunk_delete_all(repo, Star, force=force)
_chunk_delete_all(repo, AccessToken, force=force)
_chunk_delete_all(repo, RepositoryNotification, force=force)
_chunk_delete_all(repo, BlobUpload, force=force)
_chunk_delete_all(repo, RepoMirrorConfig, force=force)
_chunk_delete_all(repo, RepositoryAuthorizedEmail, force=force)
# Delete any marker rows for the repository.
DeletedRepository.delete().where(
DeletedRepository.repository == repo).execute()
# Delete the rest of the repository metadata.
try:
# Make sure the repository still exists.
fetched = Repository.get(id=repo.id)
except Repository.DoesNotExist:
return False
fetched.delete_instance(recursive=True, delete_nullable=False, force=force)
return True
def get_expired_tag(repository_id, tag_name):
""" Returns a tag with the given name that is expired in the repository or None if none.
"""
try:
return (Tag.select().where(
Tag.name == tag_name, Tag.repository == repository_id).where(
~(Tag.lifetime_end_ms >> None)).where(
Tag.lifetime_end_ms <= get_epoch_timestamp_ms()).get())
except Tag.DoesNotExist:
return None
def lookup_unrecoverable_tags(repo):
""" Returns the tags in a repository that are expired and past their time machine recovery
period. """
expired_clause = get_epoch_timestamp_ms() - (
Namespace.removed_tag_expiration_s * 1000)
return (Tag.select().join(Repository).join(
Namespace, on=(Repository.namespace_user == Namespace.id)).where(
Tag.repository == repo).where(
~(Tag.lifetime_end_ms >> None),
Tag.lifetime_end_ms <= expired_clause))
def set_tag_expiration_for_manifest(manifest_id, expiration_datetime):
""" Sets the tag expiration for any tags that point to the given manifest ID. """
query = Tag.select().where(Tag.manifest == manifest_id)
query = filter_to_alive_tags(query)
tags = list(query)
for tag in tags:
assert not tag.hidden
change_tag_expiration(tag, expiration_datetime)
return tags
def test_get_tag(initialized_db):
found = False
for tag in filter_to_visible_tags(filter_to_alive_tags(Tag.select())):
repo = tag.repository
with assert_query_count(1):
assert get_tag(repo, tag.name) == tag
found = True
assert found
def _check_manifest_used(manifest_id):
assert manifest_id is not None
with db_transaction():
# Check if the manifest is referenced by any other tag.
try:
Tag.select().where(Tag.manifest == manifest_id).get()
return True
except Tag.DoesNotExist:
pass
# Check if the manifest is referenced as a child of another manifest.
try:
ManifestChild.select().where(ManifestChild.child_manifest == manifest_id).get()
return True
except ManifestChild.DoesNotExist:
pass
return False
def test_lookup_manifest_dead_tag(initialized_db):
dead_tag = Tag.select().where(
Tag.lifetime_end_ms <= get_epoch_timestamp_ms()).get()
assert dead_tag.lifetime_end_ms <= get_epoch_timestamp_ms()
assert lookup_manifest(dead_tag.repository,
dead_tag.manifest.digest) is None
assert (lookup_manifest(dead_tag.repository,
dead_tag.manifest.digest,
allow_dead=True) == dead_tag.manifest)
def tags_containing_legacy_image(image):
""" Yields all alive Tags containing the given image as a legacy image, somewhere in its
legacy image hierarchy.
"""
ancestors_str = '%s%s/%%' % (image.ancestors, image.id)
tags = (Tag.select().join(Repository).switch(Tag).join(Manifest).join(
ManifestLegacyImage).join(Image).where(
Tag.repository == image.repository_id).where(
Image.repository == image.repository_id).where((
Image.id == image.id) | (Image.ancestors**ancestors_str)))
return filter_to_alive_tags(tags)
def set_tag_expiration_sec_for_manifest(manifest_id, expiration_seconds):
""" Sets the tag expiration for any tags that point to the given manifest ID. """
query = Tag.select().where(Tag.manifest == manifest_id)
query = filter_to_alive_tags(query)
tags = list(query)
for tag in tags:
assert not tag.hidden
set_tag_end_ms(tag,
tag.lifetime_start_ms + (expiration_seconds * 1000))
return tags
def tag_names_for_manifest(manifest_id, limit=None):
"""
Returns the names of the tags pointing to the given manifest.
"""
query = Tag.select(Tag.id, Tag.name).where(Tag.manifest == manifest_id)
if limit is not None:
query = query.limit(limit)
return [tag.name for tag in filter_to_alive_tags(query)]
def get_most_recent_tag_lifetime_start(repository_ids):
""" Returns a map from repo ID to the timestamp of the most recently pushed alive tag
for each specified repository or None if none.
"""
assert len(repository_ids) > 0 and None not in repository_ids
query = (Tag.select(Tag.repository, fn.Max(Tag.lifetime_start_ms)).where(
Tag.repository << [repo_id for repo_id in repository_ids]).group_by(
Tag.repository))
tuples = filter_to_alive_tags(query).tuples()
return {repo_id: timestamp for repo_id, timestamp in tuples}
def get_current_tag(repository_id, tag_name):
"""
Returns the current tag with the given name for the given repository.
The current tag is the tag with the highest lifetime_end_ms, regardless of
the tag being expired or hidden.
"""
try:
return (Tag.select(Tag, Manifest).join(Manifest).where(
Tag.repository == repository_id).where(
Tag.name == tag_name).order_by(-Tag.lifetime_end_ms).get())
except Tag.DoesNotExist:
return None
def test_renew_manifest_and_parent_tag_when_manifest_is_child_of_manifest_list(
self, create_repo, proxy_manifest_response):
repo_ref = create_repo(self.orgname, self.upstream_repository,
self.user)
input_list = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(UBI8_LATEST_MANIFEST_LIST),
DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE,
sparse_manifest_support=True,
)
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock()):
proxy_model = ProxyModel(
self.orgname,
self.upstream_repository,
self.user,
)
manifest_list, tag = proxy_model._create_manifest_and_retarget_tag(
repo_ref, input_list, "latest")
assert manifest_list is not None
child = (ManifestChild.select(ManifestChild.child_manifest_id).join(
Manifest,
on=(ManifestChild.child_manifest_id == Manifest.id
)).where((ManifestChild.manifest_id == manifest_list.id)
& (Manifest.digest == UBI8_LATEST_DIGEST)))
manifest_tag = Tag.select().where(Tag.manifest == child).get()
manifest_list_tag = tag
proxy_mock = proxy_manifest_response(
UBI8_LATEST_DIGEST, UBI8_LATEST_MANIFEST_SCHEMA2,
DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE)
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
proxy_model = ProxyModel(
self.orgname,
self.upstream_repository,
self.user,
)
manifest = proxy_model.lookup_manifest_by_digest(
repo_ref, UBI8_LATEST_DIGEST)
updated_tag = oci.tag.get_tag_by_manifest_id(repo_ref.id, manifest.id)
updated_list_tag = oci.tag.get_tag_by_manifest_id(
repo_ref.id, manifest_list.id)
assert updated_tag.id == manifest_tag.id
assert updated_list_tag.id == manifest_list_tag.id
assert updated_tag.lifetime_end_ms > manifest_tag.lifetime_end_ms
assert updated_list_tag.lifetime_end_ms > manifest_list_tag.lifetime_end_ms
def test_delete_tag(initialized_db):
found = False
for tag in list(filter_to_visible_tags(filter_to_alive_tags(Tag.select()))):
repo = tag.repository
assert get_tag(repo, tag.name) == tag
assert tag.lifetime_end_ms is None
with assert_query_count(3):
assert delete_tag(repo, tag.name) == tag
assert get_tag(repo, tag.name) is None
found = True
assert found
def lookup_alive_tags_shallow(repository_id, start_pagination_id=None, limit=None):
"""
Returns a list of the tags alive in the specified repository. Note that the tags returned.
*only* contain their ID and name. Also note that the Tags are returned ordered by ID.
"""
query = Tag.select(Tag.id, Tag.name).where(Tag.repository == repository_id).order_by(Tag.id)
if start_pagination_id is not None:
query = query.where(Tag.id >= start_pagination_id)
if limit is not None:
query = query.limit(limit)
return filter_to_alive_tags(query)
