def lookup_manifest_by_digest(self,
repository_ref,
manifest_digest,
allow_dead=False,
include_legacy_image=False,
require_available=False):
""" Looks up the manifest with the given digest under the given repository and returns it
or None if none. """
repo = model.repository.lookup_repository(repository_ref._db_id)
if repo is None:
return None
try:
tag_manifest = model.tag.load_manifest_by_digest(
repo.namespace_user.username,
repo.name,
manifest_digest,
allow_dead=allow_dead)
except model.tag.InvalidManifestException:
return None
legacy_image = None
if include_legacy_image:
legacy_image = self.get_legacy_image(
repository_ref,
tag_manifest.tag.image.docker_image_id,
include_parents=True)
return Manifest.for_tag_manifest(tag_manifest, legacy_image)
def get_manifest_for_tag(self, tag, backfill_if_necessary=False, include_legacy_image=False):
""" Returns the manifest associated with the given tag. """
legacy_image = None
if include_legacy_image:
legacy_image = oci.shared.get_legacy_image_for_manifest(tag._manifest)
return Manifest.for_manifest(tag._manifest, LegacyImage.for_image(legacy_image))
def get_legacy_tags_map(self, repository_ref, storage):
"""
Returns a map from tag name to its legacy image ID, for all tags with legacy images in the
repository.
Note that this can be a *very* heavy operation.
"""
tags = oci.tag.list_alive_tags(repository_ref._db_id)
legacy_images_map = oci.tag.get_legacy_images_for_tags(tags)
tags_map = {}
for tag in tags:
legacy_image = legacy_images_map.get(tag.id)
if legacy_image is not None:
tags_map[tag.name] = legacy_image.docker_image_id
else:
manifest = Manifest.for_manifest(tag.manifest, None)
if (
legacy_image is None
and manifest.media_type == DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE
):
# See if we can lookup a schema1 legacy image.
v1_compatible = self.get_schema1_parsed_manifest(manifest, "", "", "", storage)
if v1_compatible is not None:
v1_id = v1_compatible.leaf_layer_v1_image_id
if v1_id is not None:
tags_map[tag.name] = v1_id
return tags_map
def lookup_manifest_by_digest(
self,
repository_ref,
manifest_digest,
allow_dead=False,
include_legacy_image=False,
require_available=False,
):
"""
Looks up the manifest with the given digest under the given repository and returns it or
None if none.
"""
manifest = oci.manifest.lookup_manifest(
repository_ref._db_id,
manifest_digest,
allow_dead=allow_dead,
require_available=require_available,
)
if manifest is None:
return None
legacy_image = None
if include_legacy_image:
try:
legacy_image_id = database.ManifestLegacyImage.get(
manifest=manifest
).image.docker_image_id
legacy_image = self.get_legacy_image(
repository_ref, legacy_image_id, include_parents=True
)
except database.ManifestLegacyImage.DoesNotExist:
pass
return Manifest.for_manifest(manifest, legacy_image)
def create_manifest_and_retarget_tag(self,
repository_ref,
manifest_interface_instance,
tag_name,
storage,
raise_on_error=False):
"""
Creates a manifest in a repository, adding all of the necessary data in the model.
The `manifest_interface_instance` parameter must be an instance of the manifest
interface as returned by the image/docker package.
Note that all blobs referenced by the manifest must exist under the repository or this
method will fail and return None.
Returns a reference to the (created manifest, tag) or (None, None) on error, unless
raise_on_error is set to True, in which case a CreateManifestException may also be
raised.
"""
with db_disallow_replica_use():
# Get or create the manifest itself.
created_manifest = oci.manifest.get_or_create_manifest(
repository_ref._db_id,
manifest_interface_instance,
storage,
for_tagging=True,
raise_on_error=raise_on_error,
)
if created_manifest is None:
return (None, None)
# Re-target the tag to it.
tag = oci.tag.retarget_tag(
tag_name,
created_manifest.manifest,
raise_on_error=raise_on_error,
)
if tag is None:
return (None, None)
wrapped_manifest = Manifest.for_manifest(
created_manifest.manifest, self._legacy_image_id_handler)
# Apply any labels that should modify the created tag.
if created_manifest.labels_to_apply:
for key, value in created_manifest.labels_to_apply.items():
apply_label_to_manifest(dict(key=key, value=value),
wrapped_manifest, self)
# Reload the tag in case any updates were applied.
tag = database.Tag.get(id=tag.id)
return (
wrapped_manifest,
Tag.for_tag(tag,
self._legacy_image_id_handler,
manifest_row=created_manifest.manifest),
)
def find_manifests_for_sec_notification(self, manifest_digest):
"""
Finds all manifests with the given digest that live in repositories that have
registered security notifications.
"""
found = model.oci.manifest.find_manifests_for_sec_notification(manifest_digest)
for manifest in found:
yield Manifest.for_manifest(manifest, self._legacy_image_id_handler)
def _resolve_legacy_image_id(self, legacy_image_id):
"""Decodes the given legacy image ID and returns the manifest to which it points,
as well as the layer index for the image. If invalid, or the manifest was not found,
returns (None, None).
"""
manifest, layer_index = self._resolve_legacy_image_id_to_manifest_row(legacy_image_id)
if manifest is None:
return (None, None)
return Manifest.for_manifest(manifest, self._legacy_image_id_handler), layer_index
def backfill_manifest_for_tag(self, tag):
"""
Backfills a manifest for the V1 tag specified. If a manifest already exists for the tag,
returns that manifest.
NOTE: This method will only be necessary until we've completed the backfill, at which point
it should be removed.
"""
# Ensure that there isn't already a manifest for the tag.
tag_manifest = model.tag.get_tag_manifest(tag._db_id)
if tag_manifest is not None:
return Manifest.for_tag_manifest(tag_manifest)
# Create the manifest.
try:
tag_obj = database.RepositoryTag.get(id=tag._db_id)
except database.RepositoryTag.DoesNotExist:
return None
assert not tag_obj.hidden
repo = tag_obj.repository
# Write the manifest to the DB.
manifest = self._build_manifest_for_legacy_image(
tag_obj.name, tag_obj.image)
if manifest is None:
return None
blob_query = self._lookup_repo_storages_by_content_checksum(
repo, manifest.checksums)
storage_map = {blob.content_checksum: blob.id for blob in blob_query}
try:
tag_manifest = model.tag.associate_generated_tag_manifest_with_tag(
tag_obj, manifest, storage_map)
assert tag_manifest
except IntegrityError:
tag_manifest = model.tag.get_tag_manifest(tag_obj)
return Manifest.for_tag_manifest(tag_manifest)
def backfill_manifest_for_tag(self, tag):
""" Backfills a manifest for the V1 tag specified.
If a manifest already exists for the tag, returns that manifest.
NOTE: This method will only be necessary until we've completed the backfill, at which point
it should be removed.
"""
# Nothing to do for OCI tags.
manifest = tag.manifest
if manifest is None:
return None
legacy_image = oci.shared.get_legacy_image_for_manifest(manifest)
return Manifest.for_manifest(manifest, LegacyImage.for_image(legacy_image))
def get_manifest_for_tag(self,
tag,
backfill_if_necessary=False,
include_legacy_image=False):
""" Returns the manifest associated with the given tag. """
try:
tag_manifest = database.TagManifest.get(tag_id=tag._db_id)
except database.TagManifest.DoesNotExist:
if backfill_if_necessary:
return self.backfill_manifest_for_tag(tag)
return None
return Manifest.for_tag_manifest(tag_manifest)
def test_raises_exception_with_docker_v2_manifest_to_v1(self):
def get_blob(layer):
content = Bytes.for_string_or_unicode(layer).as_encoded_str()
digest = str(sha256_digest(content))
blob = store_blob_record_and_temp_link(
self.orgname,
self.upstream_repository,
digest,
ImageStorageLocation.get(name="local_us"),
len(content),
120,
)
storage.put_content(["local_us"], get_layer_path(blob), content)
return blob, digest
layer1 = json.dumps({
"config": {},
"rootfs": {
"type": "layers",
"diff_ids": []
},
"history": [{}],
})
_, config_digest = get_blob(layer1)
layer2 = "hello world"
_, blob_digest = get_blob(layer2)
builder = DockerSchema2ManifestBuilder()
builder.set_config_digest(config_digest, len(layer1.encode("utf-8")))
builder.add_layer(blob_digest, len(layer2.encode("utf-8")))
manifest = builder.build()
created_manifest = get_or_create_manifest(self.repo_ref.id, manifest,
storage)
assert created_manifest is not None
proxy_model = ProxyModel(
self.orgname,
self.upstream_repository,
self.user,
)
m = ManifestType.for_manifest(created_manifest.manifest, MagicMock())
with pytest.raises(ManifestException):
proxy_model.get_schema1_parsed_manifest(
m,
self.orgname,
self.upstream_repository,
self.tag,
storage,
raise_on_error=True,
)
def get_legacy_tags_map(self, repository_ref, storage):
"""
Returns a map from tag name to its legacy image ID, for all tags in the
repository.
Note that this can be a *very* heavy operation.
"""
tags = oci.tag.list_alive_tags(repository_ref._db_id)
tags_map = {}
for tag in tags:
root_id = Manifest.for_manifest(
tag.manifest,
self._legacy_image_id_handler).legacy_image_root_id
if root_id is not None:
tags_map[tag.name] = root_id
return tags_map
def create_manifest_with_temp_tag(
self, repository_ref, manifest_interface_instance, expiration_sec, storage
):
""" Creates a manifest under the repository and sets a temporary tag to point to it.
Returns the manifest object created or None on error.
"""
# Get or create the manifest itself. get_or_create_manifest will take care of the
# temporary tag work.
created_manifest = oci.manifest.get_or_create_manifest(
repository_ref._db_id,
manifest_interface_instance,
storage,
temp_tag_expiration_sec=expiration_sec,
)
if created_manifest is None:
return None
legacy_image = oci.shared.get_legacy_image_for_manifest(created_manifest.manifest)
li = LegacyImage.for_image(legacy_image)
return Manifest.for_manifest(created_manifest.manifest, li)
def _create_manifest_with_temp_tag(
self,
repository_ref: RepositoryReference,
manifest: ManifestInterface,
manifest_ref: str | None = None,
) -> tuple[Manifest | None, Tag | None]:
with db_disallow_replica_use():
with db_transaction():
db_manifest = oci.manifest.create_manifest(
repository_ref.id, manifest)
self._recalculate_repository_size(repository_ref)
expiration = self._config.expiration_s or None
tag = Tag.for_tag(
oci.tag.create_temporary_tag_if_necessary(
db_manifest, expiration),
self._legacy_image_id_handler,
)
wrapped_manifest = Manifest.for_manifest(
db_manifest, self._legacy_image_id_handler)
if not manifest.is_manifest_list:
self._create_placeholder_blobs(manifest, db_manifest.id,
repository_ref.id)
return wrapped_manifest, tag
manifests_to_connect = []
for child in manifest.child_manifests(content_retriever=None):
m = oci.manifest.lookup_manifest(repository_ref.id,
child.digest)
if m is None:
m = oci.manifest.create_manifest(
repository_ref.id, child)
manifests_to_connect.append(m)
oci.manifest.connect_manifests(manifests_to_connect,
db_manifest, repository_ref.id)
for db_manifest in manifests_to_connect:
oci.tag.create_temporary_tag_if_necessary(
db_manifest, expiration)
return wrapped_manifest, tag
def lookup_manifest_by_digest(
self,
repository_ref,
manifest_digest,
allow_dead=False,
require_available=False,
):
"""
Looks up the manifest with the given digest under the given repository and returns it or
None if none.
"""
manifest = oci.manifest.lookup_manifest(
repository_ref._db_id,
manifest_digest,
allow_dead=allow_dead,
require_available=require_available,
)
if manifest is None:
return None
return Manifest.for_manifest(manifest, self._legacy_image_id_handler)
def create_manifest_with_temp_tag(
self, repository_ref, manifest_interface_instance, expiration_sec, storage
):
"""
Creates a manifest under the repository and sets a temporary tag to point to it.
Returns the manifest object created or None on error.
"""
with db_disallow_replica_use():
# Get or create the manifest itself. get_or_create_manifest will take care of the
# temporary tag work.
created_manifest = oci.manifest.get_or_create_manifest(
repository_ref._db_id,
manifest_interface_instance,
storage,
temp_tag_expiration_sec=expiration_sec,
)
if created_manifest is None:
return None
return Manifest.for_manifest(created_manifest.manifest, self._legacy_image_id_handler)
def _create_manifest_and_retarget_tag(
self, repository_ref: RepositoryReference,
manifest: ManifestInterface,
tag_name: str) -> tuple[Manifest | None, Tag | None]:
"""
Creates a manifest in the given repository.
Also creates placeholders for the objects referenced by the manifest.
For manifest lists, creates placeholder sub-manifests. For regular
manifests, creates placeholder blobs.
Placeholder objects will be "filled" with the objects' contents on
upcoming client requests, as part of the flow described in the OCI
distribution specification.
Returns a reference to the (created manifest, tag) or (None, None) on error.
"""
with db_disallow_replica_use():
with db_transaction():
db_manifest = oci.manifest.lookup_manifest(repository_ref.id,
manifest.digest,
allow_dead=True)
if db_manifest is None:
db_manifest = oci.manifest.create_manifest(
repository_ref.id, manifest, raise_on_error=True)
self._recalculate_repository_size(repository_ref)
if db_manifest is None:
return None, None
# 0 means a tag never expires - if we get 0 as expiration,
# we set the tag expiration to None.
expiration = self._config.expiration_s or None
tag = oci.tag.retarget_tag(
tag_name,
db_manifest,
raise_on_error=True,
expiration_seconds=expiration,
)
if tag is None:
return None, None
wrapped_manifest = Manifest.for_manifest(
db_manifest, self._legacy_image_id_handler)
wrapped_tag = Tag.for_tag(tag,
self._legacy_image_id_handler,
manifest_row=db_manifest)
if not manifest.is_manifest_list:
self._create_placeholder_blobs(manifest, db_manifest.id,
repository_ref.id)
return wrapped_manifest, wrapped_tag
manifests_to_connect = []
for child in manifest.child_manifests(content_retriever=None):
m = oci.manifest.lookup_manifest(repository_ref.id,
child.digest,
allow_dead=True)
if m is None:
m = oci.manifest.create_manifest(
repository_ref.id, child)
oci.tag.create_temporary_tag_if_necessary(
m, self._config.expiration_s or None)
try:
ManifestChild.get(manifest=db_manifest.id,
child_manifest=m.id)
except ManifestChild.DoesNotExist:
manifests_to_connect.append(m)
oci.manifest.connect_manifests(manifests_to_connect,
db_manifest, repository_ref.id)
return wrapped_manifest, wrapped_tag
def create_manifest_and_retarget_tag(self,
repository_ref,
manifest_interface_instance,
tag_name,
storage,
raise_on_error=False):
""" Creates a manifest in a repository, adding all of the necessary data in the model.
The `manifest_interface_instance` parameter must be an instance of the manifest
interface as returned by the image/docker package.
Note that all blobs referenced by the manifest must exist under the repository or this
method will fail and return None.
Returns a reference to the (created manifest, tag) or (None, None) on error.
"""
# NOTE: Only Schema1 is supported by the pre_oci_model.
assert isinstance(manifest_interface_instance, DockerSchema1Manifest)
if not manifest_interface_instance.layers:
return None, None
# Ensure all the blobs in the manifest exist.
digests = manifest_interface_instance.checksums
query = self._lookup_repo_storages_by_content_checksum(
repository_ref._db_id, digests)
blob_map = {s.content_checksum: s for s in query}
for layer in manifest_interface_instance.layers:
digest_str = str(layer.digest)
if digest_str not in blob_map:
return None, None
# Lookup all the images and their parent images (if any) inside the manifest.
# This will let us know which v1 images we need to synthesize and which ones are invalid.
docker_image_ids = list(manifest_interface_instance.legacy_image_ids)
images_query = model.image.lookup_repository_images(
repository_ref._db_id, docker_image_ids)
image_storage_map = {
i.docker_image_id: i.storage
for i in images_query
}
# Rewrite any v1 image IDs that do not match the checksum in the database.
try:
rewritten_images = manifest_interface_instance.rewrite_invalid_image_ids(
image_storage_map)
rewritten_images = list(rewritten_images)
parent_image_map = {}
for rewritten_image in rewritten_images:
if not rewritten_image.image_id in image_storage_map:
parent_image = None
if rewritten_image.parent_image_id:
parent_image = parent_image_map.get(
rewritten_image.parent_image_id)
if parent_image is None:
parent_image = model.image.get_image(
repository_ref._db_id,
rewritten_image.parent_image_id)
if parent_image is None:
return None, None
synthesized = model.image.synthesize_v1_image(
repository_ref._db_id,
blob_map[rewritten_image.content_checksum].id,
blob_map[rewritten_image.content_checksum].image_size,
rewritten_image.image_id,
rewritten_image.created,
rewritten_image.comment,
rewritten_image.command,
rewritten_image.compat_json,
parent_image,
)
parent_image_map[rewritten_image.image_id] = synthesized
except ManifestException:
logger.exception("exception when rewriting v1 metadata")
return None, None
# Store the manifest pointing to the tag.
leaf_layer_id = rewritten_images[-1].image_id
tag_manifest, newly_created = model.tag.store_tag_manifest_for_repo(
repository_ref._db_id, tag_name, manifest_interface_instance,
leaf_layer_id, blob_map)
manifest = Manifest.for_tag_manifest(tag_manifest)
# Save the labels on the manifest.
repo_tag = tag_manifest.tag
if newly_created:
has_labels = False
with self.batch_create_manifest_labels(manifest) as add_label:
if add_label is None:
return None, None
for key, value in manifest_interface_instance.layers[
-1].v1_metadata.labels.iteritems():
media_type = 'application/json' if is_json(
value) else 'text/plain'
add_label(key, value, 'manifest', media_type)
has_labels = True
# Reload the tag in case any updates were applied.
if has_labels:
repo_tag = database.RepositoryTag.get(id=repo_tag.id)
return manifest, Tag.for_repository_tag(repo_tag)
def perform_indexing(self, start_token=None):
whitelisted_namespaces = self.app.config.get(
"SECURITY_SCANNER_V4_NAMESPACE_WHITELIST", [])
try:
indexer_state = self._secscan_api.state()
except APIRequestFailure:
return None
def eligible_manifests(base_query):
return (base_query.join(Repository).join(User).where(
User.username << whitelisted_namespaces))
min_id = (start_token.min_id if start_token is not None else
Manifest.select(fn.Min(Manifest.id)).scalar())
max_id = Manifest.select(fn.Max(Manifest.id)).scalar()
if max_id is None or min_id is None or min_id > max_id:
return None
reindex_threshold = lambda: datetime.utcnow() - timedelta(
seconds=self.app.config.get("SECURITY_SCANNER_V4_REINDEX_THRESHOLD"
))
# TODO(alecmerdler): Filter out any `Manifests` that are still being uploaded
def not_indexed_query():
return (eligible_manifests(
Manifest.select()).switch(Manifest).join(
ManifestSecurityStatus,
JOIN.LEFT_OUTER).where(ManifestSecurityStatus.id >> None))
def index_error_query():
return (eligible_manifests(Manifest.select()).switch(
Manifest).join(ManifestSecurityStatus).where(
ManifestSecurityStatus.index_status == IndexStatus.FAILED,
ManifestSecurityStatus.last_indexed < reindex_threshold(),
))
def needs_reindexing_query(indexer_hash):
return (eligible_manifests(Manifest.select()).switch(
Manifest).join(ManifestSecurityStatus).where(
ManifestSecurityStatus.indexer_hash != indexer_hash,
ManifestSecurityStatus.last_indexed < reindex_threshold(),
))
# 4^log10(total) gives us a scalable batch size into the billions.
batch_size = int(4**log10(max(10, max_id - min_id)))
iterator = itertools.chain(
yield_random_entries(
not_indexed_query,
Manifest.id,
batch_size,
max_id,
min_id,
),
yield_random_entries(
index_error_query,
Manifest.id,
batch_size,
max_id,
min_id,
),
yield_random_entries(
lambda: needs_reindexing_query(indexer_state.get("state", "")),
Manifest.id,
batch_size,
max_id,
min_id,
),
)
for candidate, abt, num_remaining in iterator:
manifest = ManifestDataType.for_manifest(candidate, None)
layers = registry_model.list_manifest_layers(
manifest, self.storage, True)
logger.debug("Indexing %s/%s@%s" %
(candidate.repository.namespace_user,
candidate.repository.name, manifest.digest))
try:
(report, state) = self._secscan_api.index(manifest, layers)
except APIRequestFailure:
logger.exception(
"Failed to perform indexing, security scanner API error")
return None
with db_transaction():
ManifestSecurityStatus.delete().where(
ManifestSecurityStatus.manifest == candidate).execute()
ManifestSecurityStatus.create(
manifest=candidate,
repository=candidate.repository,
error_json=report["err"],
index_status=(IndexStatus.FAILED if report["state"]
== IndexReportState.Index_Error else
IndexStatus.COMPLETED),
indexer_hash=state,
indexer_version=IndexerVersion.V4,
metadata_json={},
)
return ScanToken(max_id + 1)
def perform_indexing(self, start_token=None):
try:
indexer_state = self._secscan_api.state()
except APIRequestFailure:
return None
min_id = (start_token.min_id if start_token is not None else
Manifest.select(fn.Min(Manifest.id)).scalar())
max_id = Manifest.select(fn.Max(Manifest.id)).scalar()
if max_id is None or min_id is None or min_id > max_id:
return None
iterator = self._get_manifest_iterator(indexer_state, min_id, max_id)
def mark_manifest_unsupported(manifest):
with db_transaction():
ManifestSecurityStatus.delete().where(
ManifestSecurityStatus.manifest == manifest._db_id,
ManifestSecurityStatus.repository ==
manifest.repository._db_id,
).execute()
ManifestSecurityStatus.create(
manifest=manifest._db_id,
repository=manifest.repository._db_id,
index_status=IndexStatus.MANIFEST_UNSUPPORTED,
indexer_hash="none",
indexer_version=IndexerVersion.V4,
metadata_json={},
)
for candidate, abt, num_remaining in iterator:
manifest = ManifestDataType.for_manifest(candidate, None)
if manifest.is_manifest_list:
mark_manifest_unsupported(manifest)
continue
layers = registry_model.list_manifest_layers(
manifest, self.storage, True)
if layers is None or len(layers) == 0:
logger.warning(
"Cannot index %s/%s@%s due to manifest being invalid (manifest has no layers)"
% (
candidate.repository.namespace_user,
candidate.repository.name,
manifest.digest,
))
mark_manifest_unsupported(manifest)
continue
logger.debug("Indexing %s/%s@%s" %
(candidate.repository.namespace_user,
candidate.repository.name, manifest.digest))
try:
(report, state) = self._secscan_api.index(manifest, layers)
except InvalidContentSent as ex:
mark_manifest_unsupported(manifest)
logger.exception(
"Failed to perform indexing, invalid content sent")
return None
except APIRequestFailure as ex:
logger.exception(
"Failed to perform indexing, security scanner API error")
return None
with db_transaction():
ManifestSecurityStatus.delete().where(
ManifestSecurityStatus.manifest == candidate).execute()
ManifestSecurityStatus.create(
manifest=candidate,
repository=candidate.repository,
error_json=report["err"],
index_status=(IndexStatus.FAILED if report["state"]
== IndexReportState.Index_Error else
IndexStatus.COMPLETED),
indexer_hash=state,
indexer_version=IndexerVersion.V4,
metadata_json={},
)
return ScanToken(max_id + 1)
def _index(self, iterator, reindex_threshold):
def mark_manifest_unsupported(manifest):
with db_transaction():
ManifestSecurityStatus.delete().where(
ManifestSecurityStatus.manifest == manifest._db_id,
ManifestSecurityStatus.repository ==
manifest.repository._db_id,
).execute()
ManifestSecurityStatus.create(
manifest=manifest._db_id,
repository=manifest.repository._db_id,
index_status=IndexStatus.MANIFEST_UNSUPPORTED,
indexer_hash="none",
indexer_version=IndexerVersion.V4,
metadata_json={},
)
def should_skip_indexing(manifest_candidate):
"""Check whether this manifest was preempted by another worker.
That would be the case if the manifest references a manifestsecuritystatus,
or if the reindex threshold is no longer valid.
"""
if getattr(manifest_candidate, "manifestsecuritystatus", None):
return manifest_candidate.manifestsecuritystatus.last_indexed >= reindex_threshold
return len(manifest_candidate.manifestsecuritystatus_set) > 0
for candidate, abt, num_remaining in iterator:
manifest = ManifestDataType.for_manifest(candidate, None)
if manifest.is_manifest_list:
mark_manifest_unsupported(manifest)
continue
layers = registry_model.list_manifest_layers(
manifest, self.storage, True)
if layers is None or len(layers) == 0:
logger.warning(
"Cannot index %s/%s@%s due to manifest being invalid (manifest has no layers)"
% (
candidate.repository.namespace_user,
candidate.repository.name,
manifest.digest,
))
mark_manifest_unsupported(manifest)
continue
if should_skip_indexing(candidate):
logger.debug("Another worker preempted this worker")
abt.set()
continue
logger.debug("Indexing manifest [%d] %s/%s@%s" % (
manifest._db_id,
candidate.repository.namespace_user,
candidate.repository.name,
manifest.digest,
))
try:
(report, state) = self._secscan_api.index(manifest, layers)
except InvalidContentSent as ex:
mark_manifest_unsupported(manifest)
logger.exception(
"Failed to perform indexing, invalid content sent")
continue
except APIRequestFailure as ex:
logger.exception(
"Failed to perform indexing, security scanner API error")
continue
if report["state"] == IndexReportState.Index_Finished:
index_status = IndexStatus.COMPLETED
elif report["state"] == IndexReportState.Index_Error:
index_status = IndexStatus.FAILED
else:
# Unknown state don't save anything
continue
with db_transaction():
ManifestSecurityStatus.delete().where(
ManifestSecurityStatus.manifest == candidate).execute()
ManifestSecurityStatus.create(
manifest=candidate,
repository=candidate.repository,
error_json=report["err"],
index_status=index_status,
indexer_hash=state,
indexer_version=IndexerVersion.V4,
metadata_json={},
)
def create_manifest_and_retarget_tag(
self, repository_ref, manifest_interface_instance, tag_name, storage, raise_on_error=False
):
"""
Creates a manifest in a repository, adding all of the necessary data in the model.
The `manifest_interface_instance` parameter must be an instance of the manifest
interface as returned by the image/docker package.
Note that all blobs referenced by the manifest must exist under the repository or this
method will fail and return None.
Returns a reference to the (created manifest, tag) or (None, None) on error, unless
raise_on_error is set to True, in which case a CreateManifestException may also be
raised.
"""
with db_disallow_replica_use():
# Get or create the manifest itself.
created_manifest = oci.manifest.get_or_create_manifest(
repository_ref._db_id,
manifest_interface_instance,
storage,
for_tagging=True,
raise_on_error=raise_on_error,
)
if created_manifest is None:
return (None, None)
wrapped_manifest = Manifest.for_manifest(
created_manifest.manifest, self._legacy_image_id_handler
)
# Optional expiration label
# NOTE: Since there is currently only one special label on a manifest that has an effect on its tags (expiration),
# it is just simpler to set that value at tag creation time (plus it saves an additional query).
# If we were to define more of these "special" labels in the future, we should use the handlers from
# data/registry_model/label_handlers.py
if not created_manifest.newly_created:
label_dict = next(
(
label.asdict()
for label in self.list_manifest_labels(
wrapped_manifest,
key_prefix="quay",
)
if label.key == LABEL_EXPIRY_KEY
),
None,
)
else:
label_dict = next(
(
dict(key=label_key, value=label_value)
for label_key, label_value in created_manifest.labels_to_apply.items()
if label_key == LABEL_EXPIRY_KEY
),
None,
)
expiration_seconds = None
if label_dict is not None:
try:
expiration_td = convert_to_timedelta(label_dict["value"])
expiration_seconds = expiration_td.total_seconds()
except ValueError:
pass
# Re-target the tag to it.
tag = oci.tag.retarget_tag(
tag_name,
created_manifest.manifest,
raise_on_error=raise_on_error,
expiration_seconds=expiration_seconds,
)
if tag is None:
return (None, None)
return (
wrapped_manifest,
Tag.for_tag(
tag, self._legacy_image_id_handler, manifest_row=created_manifest.manifest
),
)
