def _manifest_dict(manifest):
image = None
if manifest.legacy_image_if_present is not None:
image = image_dict(manifest.legacy_image, with_history=True)
layers = None
if not manifest.is_manifest_list:
layers = registry_model.list_manifest_layers(manifest, storage)
if layers is None:
logger.debug('Missing layers for manifest `%s`', manifest.digest)
abort(404)
return {
'digest':
manifest.digest,
'is_manifest_list':
manifest.is_manifest_list,
'manifest_data':
manifest.internal_manifest_bytes.as_unicode(),
'image':
image,
'layers':
([_layer_dict(lyr.layer_info, idx)
for idx, lyr in enumerate(layers)] if layers else None),
}
def _manifest_dict(manifest):
layers = None
if not manifest.is_manifest_list:
layers = registry_model.list_manifest_layers(manifest, storage)
if layers is None:
logger.debug("Missing layers for manifest `%s`", manifest.digest)
abort(404)
image = None
if manifest.legacy_image_root_id:
# NOTE: This is replicating our older response for this endpoint, but
# returns empty for the metadata fields. This is to ensure back-compat
# for callers still using the deprecated API.
image = {
"id": manifest.legacy_image_root_id,
"created": format_date(datetime.utcnow()),
"comment": "",
"command": "",
"size": 0,
"uploading": False,
"sort_index": 0,
"ancestors": "",
}
return {
"digest": manifest.digest,
"is_manifest_list": manifest.is_manifest_list,
"manifest_data": manifest.internal_manifest_bytes.as_unicode(),
"config_media_type": manifest.config_media_type,
"layers": (
[_layer_dict(lyr.layer_info, idx) for idx, lyr in enumerate(layers)] if layers else None
),
"image": image,
}
def test_vulnerability_report_incompatible_api_response(api, initialized_db):
with fake_security_scanner(incompatible=True) as security_scanner:
with pytest.raises(APIRequestFailure):
manifest = manifest_for("devtable", "simple", "latest")
layers = registry_model.list_manifest_layers(manifest, storage, True)
api.vulnerability_report(manifest.digest)
def test_vulnerability_report(api, initialized_db):
with fake_security_scanner() as security_scanner:
manifest = manifest_for("devtable", "simple", "latest")
layers = registry_model.list_manifest_layers(manifest, storage, True)
assert manifest.digest not in security_scanner.index_reports.keys()
assert api.vulnerability_report(manifest.digest) is None
api.index(manifest, layers)
report = api.vulnerability_report(manifest.digest)
assert manifest.digest in security_scanner.vulnerability_reports.keys()
assert report is not None
def verify_replication_for(namespace, repo_name, tag_name):
repo_ref = registry_model.lookup_repository(namespace, repo_name)
assert repo_ref
tag = registry_model.get_repo_tag(repo_ref, tag_name)
assert tag
manifest = registry_model.get_manifest_for_tag(tag)
assert manifest
for layer in registry_model.list_manifest_layers(manifest, storage):
if layer.blob.digest != EMPTY_LAYER_BLOB_DIGEST:
QueueItem.select().where(
QueueItem.queue_name**("%" + layer.blob.uuid + "%")).get()
return "OK"
def _manifest_dict(manifest):
layers = None
if not manifest.is_manifest_list:
layers = registry_model.list_manifest_layers(manifest, storage)
if layers is None:
logger.debug("Missing layers for manifest `%s`", manifest.digest)
abort(404)
return {
"digest": manifest.digest,
"is_manifest_list": manifest.is_manifest_list,
"manifest_data": manifest.internal_manifest_bytes.as_unicode(),
"config_media_type": manifest.config_media_type,
"layers": (
[_layer_dict(lyr.layer_info, idx) for idx, lyr in enumerate(layers)] if layers else None
),
}
def perform_indexing(self, start_token=None):
whitelisted_namespaces = self.app.config.get(
"SECURITY_SCANNER_V4_NAMESPACE_WHITELIST", [])
try:
indexer_state = self._secscan_api.state()
except APIRequestFailure:
return None
def eligible_manifests(base_query):
return (base_query.join(Repository).join(User).where(
User.username << whitelisted_namespaces))
min_id = (start_token.min_id if start_token is not None else
Manifest.select(fn.Min(Manifest.id)).scalar())
max_id = Manifest.select(fn.Max(Manifest.id)).scalar()
if max_id is None or min_id is None or min_id > max_id:
return None
reindex_threshold = lambda: datetime.utcnow() - timedelta(
seconds=self.app.config.get("SECURITY_SCANNER_V4_REINDEX_THRESHOLD"
))
# TODO(alecmerdler): Filter out any `Manifests` that are still being uploaded
def not_indexed_query():
return (eligible_manifests(
Manifest.select()).switch(Manifest).join(
ManifestSecurityStatus,
JOIN.LEFT_OUTER).where(ManifestSecurityStatus.id >> None))
def index_error_query():
return (eligible_manifests(Manifest.select()).switch(
Manifest).join(ManifestSecurityStatus).where(
ManifestSecurityStatus.index_status == IndexStatus.FAILED,
ManifestSecurityStatus.last_indexed < reindex_threshold(),
))
def needs_reindexing_query(indexer_hash):
return (eligible_manifests(Manifest.select()).switch(
Manifest).join(ManifestSecurityStatus).where(
ManifestSecurityStatus.indexer_hash != indexer_hash,
ManifestSecurityStatus.last_indexed < reindex_threshold(),
))
# 4^log10(total) gives us a scalable batch size into the billions.
batch_size = int(4**log10(max(10, max_id - min_id)))
iterator = itertools.chain(
yield_random_entries(
not_indexed_query,
Manifest.id,
batch_size,
max_id,
min_id,
),
yield_random_entries(
index_error_query,
Manifest.id,
batch_size,
max_id,
min_id,
),
yield_random_entries(
lambda: needs_reindexing_query(indexer_state.get("state", "")),
Manifest.id,
batch_size,
max_id,
min_id,
),
)
for candidate, abt, num_remaining in iterator:
manifest = ManifestDataType.for_manifest(candidate, None)
layers = registry_model.list_manifest_layers(
manifest, self.storage, True)
logger.debug("Indexing %s/%s@%s" %
(candidate.repository.namespace_user,
candidate.repository.name, manifest.digest))
try:
(report, state) = self._secscan_api.index(manifest, layers)
except APIRequestFailure:
logger.exception(
"Failed to perform indexing, security scanner API error")
return None
with db_transaction():
ManifestSecurityStatus.delete().where(
ManifestSecurityStatus.manifest == candidate).execute()
ManifestSecurityStatus.create(
manifest=candidate,
repository=candidate.repository,
error_json=report["err"],
index_status=(IndexStatus.FAILED if report["state"]
== IndexReportState.Index_Error else
IndexStatus.COMPLETED),
indexer_hash=state,
indexer_version=IndexerVersion.V4,
metadata_json={},
)
return ScanToken(max_id + 1)
def _index(self, iterator, reindex_threshold):
def mark_manifest_unsupported(manifest):
with db_transaction():
ManifestSecurityStatus.delete().where(
ManifestSecurityStatus.manifest == manifest._db_id,
ManifestSecurityStatus.repository ==
manifest.repository._db_id,
).execute()
ManifestSecurityStatus.create(
manifest=manifest._db_id,
repository=manifest.repository._db_id,
index_status=IndexStatus.MANIFEST_UNSUPPORTED,
indexer_hash="none",
indexer_version=IndexerVersion.V4,
metadata_json={},
)
def should_skip_indexing(manifest_candidate):
"""Check whether this manifest was preempted by another worker.
That would be the case if the manifest references a manifestsecuritystatus,
or if the reindex threshold is no longer valid.
"""
if getattr(manifest_candidate, "manifestsecuritystatus", None):
return manifest_candidate.manifestsecuritystatus.last_indexed >= reindex_threshold
return len(manifest_candidate.manifestsecuritystatus_set) > 0
for candidate, abt, num_remaining in iterator:
manifest = ManifestDataType.for_manifest(candidate, None)
if manifest.is_manifest_list:
mark_manifest_unsupported(manifest)
continue
layers = registry_model.list_manifest_layers(
manifest, self.storage, True)
if layers is None or len(layers) == 0:
logger.warning(
"Cannot index %s/%s@%s due to manifest being invalid (manifest has no layers)"
% (
candidate.repository.namespace_user,
candidate.repository.name,
manifest.digest,
))
mark_manifest_unsupported(manifest)
continue
if should_skip_indexing(candidate):
logger.debug("Another worker preempted this worker")
abt.set()
continue
logger.debug("Indexing manifest [%d] %s/%s@%s" % (
manifest._db_id,
candidate.repository.namespace_user,
candidate.repository.name,
manifest.digest,
))
try:
(report, state) = self._secscan_api.index(manifest, layers)
except InvalidContentSent as ex:
mark_manifest_unsupported(manifest)
logger.exception(
"Failed to perform indexing, invalid content sent")
continue
except APIRequestFailure as ex:
logger.exception(
"Failed to perform indexing, security scanner API error")
continue
if report["state"] == IndexReportState.Index_Finished:
index_status = IndexStatus.COMPLETED
elif report["state"] == IndexReportState.Index_Error:
index_status = IndexStatus.FAILED
else:
# Unknown state don't save anything
continue
with db_transaction():
ManifestSecurityStatus.delete().where(
ManifestSecurityStatus.manifest == candidate).execute()
ManifestSecurityStatus.create(
manifest=candidate,
repository=candidate.repository,
error_json=report["err"],
index_status=index_status,
indexer_hash=state,
indexer_version=IndexerVersion.V4,
metadata_json={},
)
def perform_indexing(self, start_token=None):
try:
indexer_state = self._secscan_api.state()
except APIRequestFailure:
return None
min_id = (start_token.min_id if start_token is not None else
Manifest.select(fn.Min(Manifest.id)).scalar())
max_id = Manifest.select(fn.Max(Manifest.id)).scalar()
if max_id is None or min_id is None or min_id > max_id:
return None
iterator = self._get_manifest_iterator(indexer_state, min_id, max_id)
def mark_manifest_unsupported(manifest):
with db_transaction():
ManifestSecurityStatus.delete().where(
ManifestSecurityStatus.manifest == manifest._db_id,
ManifestSecurityStatus.repository ==
manifest.repository._db_id,
).execute()
ManifestSecurityStatus.create(
manifest=manifest._db_id,
repository=manifest.repository._db_id,
index_status=IndexStatus.MANIFEST_UNSUPPORTED,
indexer_hash="none",
indexer_version=IndexerVersion.V4,
metadata_json={},
)
for candidate, abt, num_remaining in iterator:
manifest = ManifestDataType.for_manifest(candidate, None)
if manifest.is_manifest_list:
mark_manifest_unsupported(manifest)
continue
layers = registry_model.list_manifest_layers(
manifest, self.storage, True)
if layers is None or len(layers) == 0:
logger.warning(
"Cannot index %s/%s@%s due to manifest being invalid (manifest has no layers)"
% (
candidate.repository.namespace_user,
candidate.repository.name,
manifest.digest,
))
mark_manifest_unsupported(manifest)
continue
logger.debug("Indexing %s/%s@%s" %
(candidate.repository.namespace_user,
candidate.repository.name, manifest.digest))
try:
(report, state) = self._secscan_api.index(manifest, layers)
except InvalidContentSent as ex:
mark_manifest_unsupported(manifest)
logger.exception(
"Failed to perform indexing, invalid content sent")
return None
except APIRequestFailure as ex:
logger.exception(
"Failed to perform indexing, security scanner API error")
return None
with db_transaction():
ManifestSecurityStatus.delete().where(
ManifestSecurityStatus.manifest == candidate).execute()
ManifestSecurityStatus.create(
manifest=candidate,
repository=candidate.repository,
error_json=report["err"],
index_status=(IndexStatus.FAILED if report["state"]
== IndexReportState.Index_Error else
IndexStatus.COMPLETED),
indexer_hash=state,
indexer_version=IndexerVersion.V4,
metadata_json={},
)
return ScanToken(max_id + 1)
