def list_active_repo_tags(repo,
start_id=None,
limit=None,
include_images=True):
""" Returns all of the active, non-hidden tags in a repository, joined to they images
and (if present), their manifest.
"""
if include_images:
query = _tag_alive(
RepositoryTag.select(
RepositoryTag, Image, ImageStorage,
TagManifest.digest).join(Image).join(ImageStorage).where(
RepositoryTag.repository == repo,
RepositoryTag.hidden == False).switch(RepositoryTag).join(
TagManifest,
JOIN.LEFT_OUTER).order_by(RepositoryTag.id))
else:
query = _tag_alive(
RepositoryTag.select(RepositoryTag).where(
RepositoryTag.repository == repo,
RepositoryTag.hidden == False).order_by(RepositoryTag.id))
if start_id is not None:
query = query.where(RepositoryTag.id >= start_id)
if limit is not None:
query = query.limit(limit)
return query
def find_repository_with_garbage(limit_to_gc_policy_s):
expiration_timestamp = get_epoch_timestamp() - limit_to_gc_policy_s
try:
candidates = (RepositoryTag.select(
RepositoryTag.repository).join(Repository).join(
Namespace,
on=(Repository.namespace_user == Namespace.id)).where(
~(RepositoryTag.lifetime_end_ts >> None),
(RepositoryTag.lifetime_end_ts <= expiration_timestamp),
(Namespace.removed_tag_expiration_s
== limit_to_gc_policy_s),
).limit(500).distinct().alias("candidates"))
found = (RepositoryTag.select(
candidates.c.repository_id).from_(candidates).order_by(
db_random_func()).get())
if found is None:
return
return Repository.get(Repository.id == found.repository_id)
except RepositoryTag.DoesNotExist:
return None
except Repository.DoesNotExist:
return None
def get_matching_tags(docker_image_id, storage_uuid, *args):
""" Returns a query pointing to all tags that contain the image with the
given docker_image_id and storage_uuid. """
image_row = image.get_image_with_storage(docker_image_id, storage_uuid)
if image_row is None:
return RepositoryTag.select().where(
RepositoryTag.id < 0) # Empty query.
ancestors_str = "%s%s/%%" % (image_row.ancestors, image_row.id)
return _tag_alive(
RepositoryTag.select(
*args).distinct().join(Image).join(ImageStorage).where(
RepositoryTag.hidden == False).where(
(Image.id == image_row.id)
| (Image.ancestors**ancestors_str)))
def _candidates_to_backfill(self):
def missing_tmt_query():
return (
self._filter(RepositoryTag.select())
.join(TagToRepositoryTag, JOIN.LEFT_OUTER)
.where(TagToRepositoryTag.id >> None, RepositoryTag.hidden == False)
)
min_id = self._filter(RepositoryTag.select(fn.Min(RepositoryTag.id))).scalar()
max_id = self._filter(RepositoryTag.select(fn.Max(RepositoryTag.id))).scalar()
logger.info("Found candidate range %s-%s", min_id, max_id)
iterator = yield_random_entries(missing_tmt_query, RepositoryTag.id, 1000, max_id, min_id,)
return iterator
def _delete_temp_links(repo):
""" Deletes any temp links to blobs. """
for hidden in list(
RepositoryTag.select().where(RepositoryTag.hidden == True, RepositoryTag.repository == repo)
):
hidden.delete_instance()
hidden.image.delete_instance()
def test_get_matching_tag_ids_images_filtered(initialized_db):
def filter_query(query):
return query.join(Repository).where(Repository.name == "simple")
filtered_images = filter_query(
Image.select(Image, ImageStorage)
.join(RepositoryTag)
.switch(Image)
.join(ImageStorage)
.switch(Image)
)
expected_tags_query = _tag_alive(filter_query(RepositoryTag.select()))
pairs = []
for image in filtered_images:
pairs.append((image.docker_image_id, image.storage.uuid))
matching_tags = get_matching_tags_for_images(
pairs, filter_images=filter_query, filter_tags=filter_query
)
expected_tag_ids = set([tag.id for tag in expected_tags_query])
matching_tags_ids = set([tag.id for tag in matching_tags])
# Ensure every alive tag was found.
assert matching_tags_ids == expected_tag_ids
def get_min_id_for_sec_scan(version):
"""
Gets the minimum id for a security scanning.
"""
return _tag_alive(
RepositoryTag.select(fn.Min(RepositoryTag.id)).join(Image).where(
Image.security_indexed_engine < version)).scalar()
def _get_expected_tags(image):
expected_query = (
RepositoryTag.select()
.join(Image)
.where(RepositoryTag.hidden == False)
.where((Image.id == image.id) | (Image.ancestors ** ("%%/%s/%%" % image.id)))
)
return set([tag.id for tag in _tag_alive(expected_query)])
def upgrade(tables, tester, progress_reporter):
if not app.config.get("SETUP_COMPLETE", False):
return
start_id = 0
end_id = 1000
size = 1000
max_id = RepositoryTag.select(fn.Max(RepositoryTag.id)).scalar()
if max_id is None:
return
logger.info("Found maximum ID %s" % max_id)
while True:
if start_id > max_id:
break
logger.info("Checking tag range %s - %s", start_id, end_id)
r = list(
RepositoryTag.select()
.join(Repository)
.switch(RepositoryTag)
.join(TagToRepositoryTag, JOIN.LEFT_OUTER)
.where(TagToRepositoryTag.id >> None)
.where(
RepositoryTag.hidden == False,
RepositoryTag.id >= start_id,
RepositoryTag.id < end_id,
)
)
if len(r) < 1000 and size < 100000:
size *= 2
start_id = end_id
end_id = start_id + size
if not len(r):
continue
logger.info("Found %s tags to backfill", len(r))
for index, t in enumerate(r):
logger.info("Backfilling tag %s of %s", index, len(r))
backfill_tag(t)
def purge_repository(repo, force=False):
"""
Completely delete all traces of the repository.
Will return True upon complete success, and False upon partial or total failure. Garbage
collection is incremental and repeatable, so this return value does not need to be checked or
responded to.
"""
assert repo.state == RepositoryState.MARKED_FOR_DELETION or force
# Delete the repository of all Appr-referenced entries.
# Note that new-model Tag's must be deleted in *two* passes, as they can reference parent tags,
# and MySQL is... particular... about such relationships when deleting.
if repo.kind.name == "application":
ApprTag.delete().where(ApprTag.repository == repo,
~(ApprTag.linked_tag >> None)).execute()
ApprTag.delete().where(ApprTag.repository == repo).execute()
else:
# GC to remove the images and storage.
_purge_repository_contents(repo)
# Ensure there are no additional tags, manifests, images or blobs in the repository.
assert ApprTag.select().where(ApprTag.repository == repo).count() == 0
assert Tag.select().where(Tag.repository == repo).count() == 0
assert RepositoryTag.select().where(
RepositoryTag.repository == repo).count() == 0
assert Manifest.select().where(Manifest.repository == repo).count() == 0
assert ManifestBlob.select().where(
ManifestBlob.repository == repo).count() == 0
assert Image.select().where(Image.repository == repo).count() == 0
# Delete any repository build triggers, builds, and any other large-ish reference tables for
# the repository.
_chunk_delete_all(repo, RepositoryPermission, force=force)
_chunk_delete_all(repo, RepositoryBuild, force=force)
_chunk_delete_all(repo, RepositoryBuildTrigger, force=force)
_chunk_delete_all(repo, RepositoryActionCount, force=force)
_chunk_delete_all(repo, Star, force=force)
_chunk_delete_all(repo, AccessToken, force=force)
_chunk_delete_all(repo, RepositoryNotification, force=force)
_chunk_delete_all(repo, BlobUpload, force=force)
_chunk_delete_all(repo, RepoMirrorConfig, force=force)
_chunk_delete_all(repo, RepositoryAuthorizedEmail, force=force)
# Delete any marker rows for the repository.
DeletedRepository.delete().where(
DeletedRepository.repository == repo).execute()
# Delete the rest of the repository metadata.
try:
# Make sure the repository still exists.
fetched = Repository.get(id=repo.id)
except Repository.DoesNotExist:
return False
fetched.delete_instance(recursive=True, delete_nullable=False, force=force)
return True
def lookup_unrecoverable_tags(repo):
""" Returns the tags in a repository that are expired and past their time machine recovery
period. """
expired_clause = get_epoch_timestamp() - Namespace.removed_tag_expiration_s
return (RepositoryTag.select().join(Repository).join(
Namespace, on=(Repository.namespace_user == Namespace.id)).where(
RepositoryTag.repository == repo).where(
~(RepositoryTag.lifetime_end_ts >> None),
RepositoryTag.lifetime_end_ts <= expired_clause,
))
def test_get_matching_tag_ids_for_all_images(max_subqueries, max_image_lookup_count, initialized_db):
with patch('data.model.tag._MAX_SUB_QUERIES', max_subqueries):
with patch('data.model.tag._MAX_IMAGE_LOOKUP_COUNT', max_image_lookup_count):
pairs = []
for image in Image.select(Image, ImageStorage).join(ImageStorage):
pairs.append((image.docker_image_id, image.storage.uuid))
expected_tags_ids = set([tag.id for tag in _tag_alive(RepositoryTag.select())])
matching_tags_ids = set([tag.id for tag in get_matching_tags_for_images(pairs)])
# Ensure every alive tag was found.
assert matching_tags_ids == expected_tags_ids
def upgrade_progress():
total_tags = RepositoryTag.select().where(RepositoryTag.hidden == False).count()
if total_tags == 0:
return jsonify({
'progress': 1.0,
'tags_remaining': 0,
'total_tags': 0,
})
upgraded_tags = TagToRepositoryTag.select().count()
return jsonify({
'progress': float(upgraded_tags) / total_tags,
'tags_remaining': total_tags - upgraded_tags,
'total_tags': total_tags,
})
def delete_tag(namespace_name, repository_name, tag_name, now_ms=None):
now_ms = now_ms or get_epoch_timestamp_ms()
now_ts = int(now_ms / 1000)
with db_transaction():
try:
query = _tag_alive(
RepositoryTag.select(
RepositoryTag, Repository).join(Repository).join(
Namespace,
on=(Repository.namespace_user == Namespace.id)).where(
Repository.name == repository_name,
Namespace.username == namespace_name,
RepositoryTag.name == tag_name,
),
now_ts,
)
found = db_for_update(query).get()
except RepositoryTag.DoesNotExist:
msg = "Invalid repository tag '%s' on repository '%s/%s'" % (
tag_name,
namespace_name,
repository_name,
)
raise DataModelException(msg)
found.lifetime_end_ts = now_ts
found.save()
try:
oci_tag_query = TagToRepositoryTag.select().where(
TagToRepositoryTag.repository_tag == found)
oci_tag = db_for_update(oci_tag_query).get().tag
oci_tag.lifetime_end_ms = now_ms
oci_tag.save()
except TagToRepositoryTag.DoesNotExist:
pass
return found
def list_repository_tags(namespace_name,
repository_name,
include_hidden=False,
include_storage=False):
to_select = (RepositoryTag, Image)
if include_storage:
to_select = (RepositoryTag, Image, ImageStorage)
query = _tag_alive(
RepositoryTag.select(*to_select).join(Repository).join(
Namespace, on=(Repository.namespace_user == Namespace.id
)).switch(RepositoryTag).join(Image).where(
Repository.name == repository_name,
Namespace.username == namespace_name))
if not include_hidden:
query = query.where(RepositoryTag.hidden == False)
if include_storage:
query = query.switch(Image).join(ImageStorage)
return query
def get_max_id_for_sec_scan():
""" Gets the maximum id for security scanning """
return RepositoryTag.select(fn.Max(RepositoryTag.id)).scalar()
