def test_repository_tag_history(namespace, name, expected_tag_count,
has_expired, registry_model,
with_size_fallback):
# Pre-cache media type loads to ensure consistent query count.
Manifest.media_type.get_name(1)
# If size fallback is requested, delete the sizes on the manifest rows.
if with_size_fallback:
Manifest.update(layers_compressed_size=None).execute()
repository_ref = registry_model.lookup_repository(namespace, name)
with assert_query_count(2 if with_size_fallback else 1):
history, has_more = registry_model.list_repository_tag_history(
repository_ref)
assert not has_more
assert len(history) == expected_tag_count
for tag in history:
# Retrieve the manifest to ensure it doesn't issue extra queries.
tag.manifest
# Verify that looking up the size doesn't issue extra queries.
tag.manifest_layers_size
if has_expired:
# Ensure the latest tag is marked expired, since there is an expired one.
with assert_query_count(1):
assert registry_model.has_expired_tag(repository_ref, "latest")
def test_perform_indexing_manifest_list(initialized_db, set_secscan_config):
repository_ref = registry_model.lookup_repository("devtable", "simple")
tag = registry_model.get_repo_tag(repository_ref, "latest")
manifest = registry_model.get_manifest_for_tag(tag)
Manifest.update(media_type=MediaType.get(
name=DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE)).execute()
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan.perform_indexing()
assert ManifestSecurityStatus.select().count() == Manifest.select().count()
for mss in ManifestSecurityStatus.select():
assert mss.index_status == IndexStatus.MANIFEST_UNSUPPORTED
def _backfill_manifests(self):
try:
Manifest.select().where(
Manifest.layers_compressed_size >> None).get()
except Manifest.DoesNotExist:
logger.debug("Manifest backfill worker has completed; skipping")
return False
iterator = yield_random_entries(
lambda: Manifest.select().where(Manifest.layers_compressed_size >>
None),
Manifest.id,
250,
Manifest.select(fn.Max(Manifest.id)).scalar(),
1,
)
for manifest_row, abt, _ in iterator:
if manifest_row.layers_compressed_size is not None:
logger.debug("Another worker preempted this worker")
abt.set()
continue
logger.debug("Setting layers compressed size for manifest %s",
manifest_row.id)
layers_compressed_size = -1
config_media_type = None
manifest_bytes = Bytes.for_string_or_unicode(
manifest_row.manifest_bytes)
try:
parsed = parse_manifest_from_bytes(
manifest_bytes,
manifest_row.media_type.name,
validate=False)
layers_compressed_size = parsed.layers_compressed_size
if layers_compressed_size is None:
layers_compressed_size = 0
config_media_type = parsed.config_media_type or None
except ManifestException as me:
logger.warning(
"Got exception when trying to parse manifest %s: %s",
manifest_row.id, me)
assert layers_compressed_size is not None
updated = (Manifest.update(
layers_compressed_size=layers_compressed_size,
config_media_type=config_media_type,
).where(Manifest.id == manifest_row.id,
Manifest.layers_compressed_size >> None).execute())
if updated != 1:
logger.debug("Another worker preempted this worker")
abt.set()
continue
return True
def _update_manifest_for_tag(
self,
repo_ref: RepositoryReference,
tag: Tag,
manifest: Manifest,
manifest_ref: str,
create_manifest_fn,
) -> tuple[Tag, bool]:
"""
Updates a placeholder manifest with the given tag name.
If the manifest is stale, downloads it from the upstream registry
and creates a new manifest and retargets the tag.
A manifest is considered stale when the manifest's digest changed in
the upstream registry.
A manifest is considered a placeholder when its db entry exists, but
its manifest_bytes field is empty.
Raises UpstreamRegistryError if the upstream registry returns anything
other than 200.
Raises ManifestDoesNotExist if the given manifest was not found in the
database.
Returns a new tag if one was created, or the existing one with a manifest
freshly out of the database, and a boolean indicating whether the returned
tag was newly created or not.
"""
upstream_manifest = None
upstream_digest = self._proxy.manifest_exists(manifest_ref,
ACCEPTED_MEDIA_TYPES)
up_to_date = manifest.digest == upstream_digest
# manifest_exists will return an empty/None digest when the upstream
# registry omits the docker-content-digest header.
if not upstream_digest:
upstream_manifest = self._pull_upstream_manifest(
repo_ref.name, manifest_ref)
up_to_date = manifest.digest == upstream_manifest.digest
placeholder = manifest.internal_manifest_bytes.as_unicode() == ""
if up_to_date and not placeholder:
return tag, False
if upstream_manifest is None:
upstream_manifest = self._pull_upstream_manifest(
repo_ref.name, manifest_ref)
self._enforce_repository_quota(repo_ref)
if up_to_date and placeholder:
with db_disallow_replica_use():
with db_transaction():
q = ManifestTable.update(
manifest_bytes=upstream_manifest.bytes.as_unicode(),
layers_compressed_size=upstream_manifest.
layers_compressed_size,
).where(ManifestTable.id == manifest.id)
q.execute()
self._create_placeholder_blobs(upstream_manifest,
manifest.id, repo_ref.id)
db_tag = oci.tag.get_tag_by_manifest_id(
repo_ref.id, manifest.id)
self._recalculate_repository_size(repo_ref)
return Tag.for_tag(db_tag,
self._legacy_image_id_handler), False
# if we got here, the manifest is stale, so we both create a new manifest
# entry in the db, and retarget the tag.
_, tag = create_manifest_fn(repo_ref, upstream_manifest, manifest_ref)
return tag, True
