def test_perform_indexing_api_request_failure_index(initialized_db,
set_secscan_config):
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.return_value = {"state": "abc"}
secscan._secscan_api.index.side_effect = APIRequestFailure()
next_token = secscan.perform_indexing()
assert next_token is None
assert ManifestSecurityStatus.select().count() == 0
# Set security scanner to return good results and attempt indexing again
secscan._secscan_api.index.side_effect = None
secscan._secscan_api.index.return_value = (
{
"err": None,
"state": IndexReportState.Index_Finished
},
"abc",
)
next_token = secscan.perform_indexing()
assert next_token.min_id == Manifest.select(fn.Max(
Manifest.id)).scalar() + 1
assert ManifestSecurityStatus.select().count() == Manifest.select(
fn.Max(Manifest.id)).count()
def test_load_security_information_success(initialized_db, set_secscan_config):
repository_ref = registry_model.lookup_repository("devtable", "simple")
tag = registry_model.get_repo_tag(repository_ref, "latest")
manifest = registry_model.get_manifest_for_tag(tag)
ManifestSecurityStatus.create(
manifest=manifest._db_id,
repository=repository_ref._db_id,
error_json={},
index_status=IndexStatus.COMPLETED,
indexer_hash="abc",
indexer_version=IndexerVersion.V4,
metadata_json={},
)
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.vulnerability_report.return_value = {
"manifest_hash": manifest.digest,
"state": "IndexFinished",
"packages": {},
"distributions": {},
"repository": {},
"environments": {},
"package_vulnerabilities": {},
"success": True,
"err": "",
}
result = secscan.load_security_information(manifest)
assert result.status == ScanLookupStatus.SUCCESS
assert result.security_information == SecurityInformation(
Layer(manifest.digest, "", "", 4, []))
def test_perform_indexing_needs_reindexing_within_reindex_threshold(
initialized_db, set_secscan_config):
app.config["SECURITY_SCANNER_V4_REINDEX_THRESHOLD"] = 300
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.return_value = {"state": "xyz"}
secscan._secscan_api.index.return_value = (
{
"err": None,
"state": IndexReportState.Index_Finished
},
"xyz",
)
for manifest in Manifest.select():
ManifestSecurityStatus.create(
manifest=manifest,
repository=manifest.repository,
error_json={},
index_status=IndexStatus.COMPLETED,
indexer_hash="abc",
indexer_version=IndexerVersion.V4,
metadata_json={},
)
secscan.perform_indexing()
assert ManifestSecurityStatus.select().count() == Manifest.select().count()
for mss in ManifestSecurityStatus.select():
assert mss.indexer_hash == "abc"
def test_perform_indexing_whitelist(initialized_db, set_secscan_config):
app.config["SECURITY_SCANNER_V4_NAMESPACE_WHITELIST"] = ["devtable"]
expected_manifests = (Manifest.select().join(Repository).join(User).where(
User.username == "devtable"))
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.return_value = {"state": "abc"}
secscan._secscan_api.index.return_value = (
{
"err": None,
"state": IndexReportState.Index_Finished
},
"abc",
)
next_token = secscan.perform_indexing()
assert secscan._secscan_api.index.call_count == expected_manifests.count()
for mss in ManifestSecurityStatus.select():
assert mss.repository.namespace_user.username == "devtable"
assert ManifestSecurityStatus.select().count() == expected_manifests.count(
)
assert (
Manifest.get_by_id(next_token.min_id -
1).repository.namespace_user.username == "devtable")
def test_perform_indexing_api_request_failure_index(initialized_db):
app.config["SECURITY_SCANNER_V4_NAMESPACE_WHITELIST"] = ["devtable"]
expected_manifests = (
Manifest.select(fn.Max(Manifest.id))
.join(Repository)
.join(User)
.where(User.username == "devtable")
)
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.return_value = "abc"
secscan._secscan_api.index.side_effect = APIRequestFailure()
next_token = secscan.perform_indexing()
assert next_token is None
assert ManifestSecurityStatus.select().count() == 0
# Set security scanner to return good results and attempt indexing again
secscan._secscan_api.index.side_effect = None
secscan._secscan_api.index.return_value = (
{"err": None, "state": IndexReportState.Index_Finished},
"abc",
)
next_token = secscan.perform_indexing()
assert next_token.min_id == expected_manifests.scalar() + 1
assert ManifestSecurityStatus.select().count() == expected_manifests.count()
def test_perform_indexing_needs_reindexing_skip_unsupported(
initialized_db, set_secscan_config):
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.return_value = {"state": "new hash"}
secscan._secscan_api.index.return_value = (
{
"err": None,
"state": IndexReportState.Index_Finished
},
"new hash",
)
for manifest in Manifest.select():
ManifestSecurityStatus.create(
manifest=manifest,
repository=manifest.repository,
error_json={},
index_status=IndexStatus.MANIFEST_UNSUPPORTED,
indexer_hash="old hash",
indexer_version=IndexerVersion.V4,
last_indexed=datetime.utcnow() - timedelta(
seconds=app.config["SECURITY_SCANNER_V4_REINDEX_THRESHOLD"] +
60),
metadata_json={},
)
secscan.perform_indexing()
# Since this manifest should not be scanned, the old hash should remain
assert ManifestSecurityStatus.select().count() == Manifest.select().count()
for mss in ManifestSecurityStatus.select():
assert mss.indexer_hash == "old hash"
def test_manifest_iterator(initialized_db, set_secscan_config, index_status,
indexer_state, seconds, expect_zero):
secscan = V4SecurityScanner(app, instance_keys, storage)
for manifest in Manifest.select():
with db_transaction():
ManifestSecurityStatus.delete().where(
ManifestSecurityStatus.manifest == manifest,
ManifestSecurityStatus.repository == manifest.repository,
).execute()
ManifestSecurityStatus.create(
manifest=manifest,
repository=manifest.repository,
error_json={},
index_status=index_status,
indexer_hash="old hash",
indexer_version=IndexerVersion.V4,
last_indexed=datetime.utcnow() - timedelta(seconds=seconds),
metadata_json={},
)
iterator = secscan._get_manifest_iterator(
indexer_state,
Manifest.select(fn.Min(Manifest.id)).scalar(),
Manifest.select(fn.Max(Manifest.id)).scalar(),
)
count = 0
for candidate, abt, num_remaining in iterator:
count = count + 1
if expect_zero:
assert count == 0
else:
assert count != 0
def test_load_security_information_api_request_failure(initialized_db,
set_secscan_config):
repository_ref = registry_model.lookup_repository("devtable", "simple")
tag = registry_model.get_repo_tag(repository_ref,
"latest",
include_legacy_image=True)
manifest = registry_model.get_manifest_for_tag(tag,
backfill_if_necessary=True)
mss = ManifestSecurityStatus.create(
manifest=manifest._db_id,
repository=repository_ref._db_id,
error_json={},
index_status=IndexStatus.COMPLETED,
indexer_hash="abc",
indexer_version=IndexerVersion.V4,
metadata_json={},
)
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.vulnerability_report.side_effect = APIRequestFailure()
assert secscan.load_security_information(
manifest).status == ScanLookupStatus.COULD_NOT_LOAD
assert not ManifestSecurityStatus.select().where(
ManifestSecurityStatus.id == mss.id).exists()
def test_load_security_information_api_returns_none(initialized_db,
set_secscan_config):
repository_ref = registry_model.lookup_repository("devtable", "simple")
tag = registry_model.get_repo_tag(repository_ref,
"latest",
include_legacy_image=True)
manifest = registry_model.get_manifest_for_tag(tag,
backfill_if_necessary=True)
ManifestSecurityStatus.create(
manifest=manifest._db_id,
repository=repository_ref._db_id,
error_json={},
index_status=IndexStatus.COMPLETED,
indexer_hash="abc",
indexer_version=IndexerVersion.V4,
metadata_json={},
)
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.vulnerability_report.return_value = None
assert secscan.load_security_information(
manifest).status == ScanLookupStatus.NOT_YET_INDEXED
def test_perform_indexing_failed_within_reindex_threshold(
initialized_db, set_secscan_config):
app.config["SECURITY_SCANNER_V4_REINDEX_THRESHOLD"] = 300
expected_manifests = (Manifest.select().join(Repository).join(User).where(
User.username == "devtable"))
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.return_value = {"state": "abc"}
secscan._secscan_api.index.return_value = (
{
"err": None,
"state": IndexReportState.Index_Finished
},
"abc",
)
for manifest in expected_manifests:
ManifestSecurityStatus.create(
manifest=manifest,
repository=manifest.repository,
error_json={},
index_status=IndexStatus.FAILED,
indexer_hash="abc",
indexer_version=IndexerVersion.V4,
metadata_json={},
)
secscan.perform_indexing()
assert ManifestSecurityStatus.select().count() == expected_manifests.count(
)
for mss in ManifestSecurityStatus.select():
assert mss.index_status == IndexStatus.FAILED
def test_perform_indexing_needs_reindexing(initialized_db):
app.config["SECURITY_SCANNER_V4_NAMESPACE_WHITELIST"] = ["devtable"]
expected_manifests = (
Manifest.select().join(Repository).join(User).where(User.username == "devtable")
)
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.return_value = "xyz"
secscan._secscan_api.index.return_value = (
{"err": None, "state": IndexReportState.Index_Finished},
"xyz",
)
for manifest in expected_manifests:
ManifestSecurityStatus.create(
manifest=manifest,
repository=manifest.repository,
error_json={},
index_status=IndexStatus.COMPLETED,
indexer_hash="abc",
indexer_version=IndexerVersion.V4,
metadata_json={},
)
secscan.perform_indexing()
assert ManifestSecurityStatus.select().count() == expected_manifests.count()
for mss in ManifestSecurityStatus.select():
assert mss.indexer_hash == "xyz"
def test_perform_indexing_failed(initialized_db, set_secscan_config):
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.return_value = {"state": "abc"}
secscan._secscan_api.index.return_value = (
{
"err": None,
"state": IndexReportState.Index_Finished
},
"abc",
)
for manifest in Manifest.select():
ManifestSecurityStatus.create(
manifest=manifest,
repository=manifest.repository,
error_json={},
index_status=IndexStatus.FAILED,
indexer_hash="abc",
indexer_version=IndexerVersion.V4,
last_indexed=datetime.utcnow() - timedelta(
seconds=app.config["SECURITY_SCANNER_V4_REINDEX_THRESHOLD"] +
60),
metadata_json={},
)
secscan.perform_indexing()
assert ManifestSecurityStatus.select().count() == Manifest.select().count()
for mss in ManifestSecurityStatus.select():
assert mss.index_status == IndexStatus.COMPLETED
def test_perform_indexing_invalid_manifest(initialized_db, set_secscan_config):
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
# Delete all ManifestBlob rows to cause the manifests to be invalid.
ManifestBlob.delete().execute()
secscan.perform_indexing()
assert ManifestSecurityStatus.select().count() == Manifest.select().count()
for mss in ManifestSecurityStatus.select():
assert mss.index_status == IndexStatus.MANIFEST_UNSUPPORTED
def test_perform_indexing_api_request_index_error_response(initialized_db, set_secscan_config):
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.return_value = {"state": "xyz"}
secscan._secscan_api.index.return_value = (
{"err": "something", "state": IndexReportState.Index_Error},
"xyz",
)
next_token = secscan.perform_indexing()
assert next_token.min_id == Manifest.select(fn.Max(Manifest.id)).scalar() + 1
assert ManifestSecurityStatus.select().count() == Manifest.select(fn.Max(Manifest.id)).count()
for mss in ManifestSecurityStatus.select():
assert mss.index_status == IndexStatus.FAILED
def test_perform_indexing_manifest_list(initialized_db, set_secscan_config):
repository_ref = registry_model.lookup_repository("devtable", "simple")
tag = registry_model.get_repo_tag(repository_ref, "latest")
manifest = registry_model.get_manifest_for_tag(tag)
Manifest.update(media_type=MediaType.get(
name=DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE)).execute()
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan.perform_indexing()
assert ManifestSecurityStatus.select().count() == Manifest.select().count()
for mss in ManifestSecurityStatus.select():
assert mss.index_status == IndexStatus.MANIFEST_UNSUPPORTED
def mark_manifest_unsupported(manifest):
with db_transaction():
ManifestSecurityStatus.delete().where(
ManifestSecurityStatus.manifest == manifest._db_id,
ManifestSecurityStatus.repository ==
manifest.repository._db_id,
).execute()
ManifestSecurityStatus.create(
manifest=manifest._db_id,
repository=manifest.repository._db_id,
index_status=IndexStatus.MANIFEST_UNSUPPORTED,
indexer_hash="none",
indexer_version=IndexerVersion.V4,
metadata_json={},
)
def test_load_security_information_failed_to_index(initialized_db):
repository_ref = registry_model.lookup_repository("devtable", "simple")
tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True)
manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
ManifestSecurityStatus.create(
manifest=manifest._db_id,
repository=repository_ref._db_id,
error_json='failed to fetch layers: encountered error while fetching a layer: fetcher: unknown content-type "binary/octet-stream"',
index_status=IndexStatus.FAILED,
indexer_hash="",
indexer_version=IndexerVersion.V4,
metadata_json={},
)
secscan = V4SecurityScanner(app, instance_keys, storage)
assert secscan.load_security_information(manifest).status == ScanLookupStatus.FAILED_TO_INDEX
def test_perform_indexing_whitelist(initialized_db, set_secscan_config):
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.return_value = {"state": "abc"}
secscan._secscan_api.index.return_value = (
{"err": None, "state": IndexReportState.Index_Finished},
"abc",
)
next_token = secscan.perform_indexing()
assert next_token.min_id == Manifest.select(fn.Max(Manifest.id)).scalar() + 1
assert secscan._secscan_api.index.call_count == Manifest.select().count()
assert ManifestSecurityStatus.select().count() == Manifest.select().count()
for mss in ManifestSecurityStatus.select():
assert mss.index_status == IndexStatus.COMPLETED
def test_perform_indexing_api_request_failure_state(initialized_db, set_secscan_config):
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.side_effect = APIRequestFailure()
next_token = secscan.perform_indexing()
assert next_token is None
assert ManifestSecurityStatus.select().count() == 0
def manifest_security_backfill_status():
manifest_count = Manifest.select().count()
mss_count = ManifestSecurityStatus.select().count()
if manifest_count == 0:
percent = 1.0
else:
percent = mss_count / float(manifest_count)
return jsonify({"backfill_percent": round(percent * 100, 2)})
def test_perform_indexing_api_request_failure_state(initialized_db):
app.config["SECURITY_SCANNER_V4_NAMESPACE_WHITELIST"] = ["devtable"]
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.side_effect = APIRequestFailure()
next_token = secscan.perform_indexing()
assert next_token is None
assert ManifestSecurityStatus.select().count() == 0
def test_perform_indexing_api_request_non_finished_state(initialized_db, set_secscan_config):
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.return_value = {"state": "xyz"}
secscan._secscan_api.index.return_value = (
{"err": "something", "state": "ScanLayers"},
"xyz",
)
next_token = secscan.perform_indexing()
assert next_token and next_token.min_id == Manifest.select(fn.Max(Manifest.id)).scalar() + 1
assert ManifestSecurityStatus.select().count() == 0
def load_security_information(self,
manifest_or_legacy_image,
include_vulnerabilities=False):
if not isinstance(manifest_or_legacy_image, ManifestDataType):
return SecurityInformationLookupResult.with_status(
ScanLookupStatus.UNSUPPORTED_FOR_INDEXING)
status = None
try:
status = ManifestSecurityStatus.get(
manifest=manifest_or_legacy_image._db_id)
except ManifestSecurityStatus.DoesNotExist:
return SecurityInformationLookupResult.with_status(
ScanLookupStatus.NOT_YET_INDEXED)
if status.index_status == IndexStatus.FAILED:
return SecurityInformationLookupResult.with_status(
ScanLookupStatus.FAILED_TO_INDEX)
if status.index_status == IndexStatus.MANIFEST_UNSUPPORTED:
return SecurityInformationLookupResult.with_status(
ScanLookupStatus.UNSUPPORTED_FOR_INDEXING)
if status.index_status == IndexStatus.IN_PROGRESS:
return SecurityInformationLookupResult.with_status(
ScanLookupStatus.NOT_YET_INDEXED)
assert status.index_status == IndexStatus.COMPLETED
try:
report = self._secscan_api.vulnerability_report(
manifest_or_legacy_image.digest)
except APIRequestFailure as arf:
try:
status.delete_instance()
except ReadOnlyModeException:
pass
return SecurityInformationLookupResult.for_request_error(str(arf))
if report is None:
return SecurityInformationLookupResult.with_status(
ScanLookupStatus.NOT_YET_INDEXED)
# TODO(alecmerdler): Provide a way to indicate the current scan is outdated (`report.state != status.indexer_hash`)
return SecurityInformationLookupResult.for_data(
SecurityInformation(
Layer(report["manifest_hash"], "", "", 4,
features_for(report))))
def test_perform_indexing_empty_whitelist(initialized_db):
app.config["SECURITY_SCANNER_V4_NAMESPACE_WHITELIST"] = []
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.return_value = "abc"
secscan._secscan_api.index.return_value = (
{"err": None, "state": IndexReportState.Index_Finished},
"abc",
)
next_token = secscan.perform_indexing()
assert secscan._secscan_api.index.call_count == 0
assert ManifestSecurityStatus.select().count() == 0
assert next_token.min_id == Manifest.select(fn.Max(Manifest.id)).scalar() + 1
def test_load_security_information(indexed_v2, indexed_v4, expected_status,
initialized_db):
secscan_model.configure(app, instance_keys, storage)
repository_ref = registry_model.lookup_repository("devtable", "simple")
tag = registry_model.find_matching_tag(repository_ref, ["latest"])
manifest = registry_model.get_manifest_for_tag(tag)
assert manifest
registry_model.populate_legacy_images_for_testing(manifest, storage)
image = shared.get_legacy_image_for_manifest(manifest._db_id)
if indexed_v2:
image.security_indexed = False
image.security_indexed_engine = 3
image.save()
else:
ManifestLegacyImage.delete().where(
ManifestLegacyImage.manifest == manifest._db_id).execute()
if indexed_v4:
ManifestSecurityStatus.create(
manifest=manifest._db_id,
repository=repository_ref._db_id,
error_json={},
index_status=IndexStatus.MANIFEST_UNSUPPORTED,
indexer_hash="abc",
indexer_version=IndexerVersion.V4,
metadata_json={},
)
result = secscan_model.load_security_information(manifest, True)
assert isinstance(result, SecurityInformationLookupResult)
assert result.status == expected_status
def purge_repository(repo, force=False):
"""
Completely delete all traces of the repository.
Will return True upon complete success, and False upon partial or total failure. Garbage
collection is incremental and repeatable, so this return value does not need to be checked or
responded to.
"""
assert repo.state == RepositoryState.MARKED_FOR_DELETION or force
# Update the repo state to ensure nothing else is written to it.
repo.state = RepositoryState.MARKED_FOR_DELETION
repo.save()
# Delete the repository of all Appr-referenced entries.
# Note that new-model Tag's must be deleted in *two* passes, as they can reference parent tags,
# and MySQL is... particular... about such relationships when deleting.
if repo.kind.name == "application":
fst_pass = (ApprTag.delete().where(
ApprTag.repository == repo,
~(ApprTag.linked_tag >> None)).execute())
snd_pass = ApprTag.delete().where(ApprTag.repository == repo).execute()
gc_table_rows_deleted.labels(table="ApprTag").inc(fst_pass + snd_pass)
else:
# GC to remove the images and storage.
_purge_repository_contents(repo)
# Ensure there are no additional tags, manifests, images or blobs in the repository.
assert ApprTag.select().where(ApprTag.repository == repo).count() == 0
assert Tag.select().where(Tag.repository == repo).count() == 0
assert RepositoryTag.select().where(
RepositoryTag.repository == repo).count() == 0
assert Manifest.select().where(Manifest.repository == repo).count() == 0
assert ManifestBlob.select().where(
ManifestBlob.repository == repo).count() == 0
assert UploadedBlob.select().where(
UploadedBlob.repository == repo).count() == 0
assert (ManifestSecurityStatus.select().where(
ManifestSecurityStatus.repository == repo).count() == 0)
assert Image.select().where(Image.repository == repo).count() == 0
# Delete any repository build triggers, builds, and any other large-ish reference tables for
# the repository.
_chunk_delete_all(repo, RepositoryPermission, force=force)
_chunk_delete_all(repo, RepositoryBuild, force=force)
_chunk_delete_all(repo, RepositoryBuildTrigger, force=force)
_chunk_delete_all(repo, RepositoryActionCount, force=force)
_chunk_delete_all(repo, Star, force=force)
_chunk_delete_all(repo, AccessToken, force=force)
_chunk_delete_all(repo, RepositoryNotification, force=force)
_chunk_delete_all(repo, BlobUpload, force=force)
_chunk_delete_all(repo, RepoMirrorConfig, force=force)
_chunk_delete_all(repo, RepositoryAuthorizedEmail, force=force)
# Delete any marker rows for the repository.
DeletedRepository.delete().where(
DeletedRepository.repository == repo).execute()
# Delete the rest of the repository metadata.
try:
# Make sure the repository still exists.
fetched = Repository.get(id=repo.id)
except Repository.DoesNotExist:
return False
try:
fetched.delete_instance(recursive=True,
delete_nullable=False,
force=force)
gc_repos_purged.inc()
return True
except IntegrityError:
return False
def _garbage_collect_manifest(manifest_id, context):
assert manifest_id is not None
# Make sure the manifest isn't referenced.
if _check_manifest_used(manifest_id):
return False
# Add the manifest's blobs to the context to be GCed.
for manifest_blob in ManifestBlob.select().where(
ManifestBlob.manifest == manifest_id):
context.add_blob_id(manifest_blob.blob_id)
# Retrieve the manifest's associated image, if any.
try:
legacy_image_id = ManifestLegacyImage.get(
manifest=manifest_id).image_id
context.add_legacy_image_id(legacy_image_id)
except ManifestLegacyImage.DoesNotExist:
legacy_image_id = None
# Add child manifests to be GCed.
for connector in ManifestChild.select().where(
ManifestChild.manifest == manifest_id):
context.add_manifest_id(connector.child_manifest_id)
# Add the labels to be GCed.
for manifest_label in ManifestLabel.select().where(
ManifestLabel.manifest == manifest_id):
context.add_label_id(manifest_label.label_id)
# Delete the manifest.
with db_transaction():
try:
manifest = Manifest.select().where(
Manifest.id == manifest_id).get()
except Manifest.DoesNotExist:
return False
assert manifest.id == manifest_id
assert manifest.repository_id == context.repository.id
if _check_manifest_used(manifest_id):
return False
# Delete any label mappings.
deleted_tag_manifest_label_map = (TagManifestLabelMap.delete().where(
TagManifestLabelMap.manifest == manifest_id).execute())
# Delete any mapping rows for the manifest.
deleted_tag_manifest_to_manifest = (
TagManifestToManifest.delete().where(
TagManifestToManifest.manifest == manifest_id).execute())
# Delete any label rows.
deleted_manifest_label = (ManifestLabel.delete().where(
ManifestLabel.manifest == manifest_id,
ManifestLabel.repository == context.repository,
).execute())
# Delete any child manifest rows.
deleted_manifest_child = (ManifestChild.delete().where(
ManifestChild.manifest == manifest_id,
ManifestChild.repository == context.repository,
).execute())
# Delete the manifest blobs for the manifest.
deleted_manifest_blob = (ManifestBlob.delete().where(
ManifestBlob.manifest == manifest_id,
ManifestBlob.repository == context.repository).execute())
# Delete the security status for the manifest
deleted_manifest_security = (ManifestSecurityStatus.delete().where(
ManifestSecurityStatus.manifest == manifest_id,
ManifestSecurityStatus.repository == context.repository,
).execute())
# Delete the manifest legacy image row.
deleted_manifest_legacy_image = 0
if legacy_image_id:
deleted_manifest_legacy_image = (
ManifestLegacyImage.delete().where(
ManifestLegacyImage.manifest == manifest_id,
ManifestLegacyImage.repository == context.repository,
).execute())
# Delete the manifest.
manifest.delete_instance()
context.mark_manifest_removed(manifest)
gc_table_rows_deleted.labels(
table="TagManifestLabelMap").inc(deleted_tag_manifest_label_map)
gc_table_rows_deleted.labels(
table="TagManifestToManifest").inc(deleted_tag_manifest_to_manifest)
gc_table_rows_deleted.labels(
table="ManifestLabel").inc(deleted_manifest_label)
gc_table_rows_deleted.labels(
table="ManifestChild").inc(deleted_manifest_child)
gc_table_rows_deleted.labels(
table="ManifestBlob").inc(deleted_manifest_blob)
gc_table_rows_deleted.labels(
table="ManifestSecurityStatus").inc(deleted_manifest_security)
if deleted_manifest_legacy_image:
gc_table_rows_deleted.labels(
table="ManifestLegacyImage").inc(deleted_manifest_legacy_image)
gc_table_rows_deleted.labels(table="Manifest").inc()
return True
def _index(self, iterator, reindex_threshold):
def mark_manifest_unsupported(manifest):
with db_transaction():
ManifestSecurityStatus.delete().where(
ManifestSecurityStatus.manifest == manifest._db_id,
ManifestSecurityStatus.repository ==
manifest.repository._db_id,
).execute()
ManifestSecurityStatus.create(
manifest=manifest._db_id,
repository=manifest.repository._db_id,
index_status=IndexStatus.MANIFEST_UNSUPPORTED,
indexer_hash="none",
indexer_version=IndexerVersion.V4,
metadata_json={},
)
def should_skip_indexing(manifest_candidate):
"""Check whether this manifest was preempted by another worker.
That would be the case if the manifest references a manifestsecuritystatus,
or if the reindex threshold is no longer valid.
"""
if getattr(manifest_candidate, "manifestsecuritystatus", None):
return manifest_candidate.manifestsecuritystatus.last_indexed >= reindex_threshold
return len(manifest_candidate.manifestsecuritystatus_set) > 0
for candidate, abt, num_remaining in iterator:
manifest = ManifestDataType.for_manifest(candidate, None)
if manifest.is_manifest_list:
mark_manifest_unsupported(manifest)
continue
layers = registry_model.list_manifest_layers(
manifest, self.storage, True)
if layers is None or len(layers) == 0:
logger.warning(
"Cannot index %s/%s@%s due to manifest being invalid (manifest has no layers)"
% (
candidate.repository.namespace_user,
candidate.repository.name,
manifest.digest,
))
mark_manifest_unsupported(manifest)
continue
if should_skip_indexing(candidate):
logger.debug("Another worker preempted this worker")
abt.set()
continue
logger.debug("Indexing manifest [%d] %s/%s@%s" % (
manifest._db_id,
candidate.repository.namespace_user,
candidate.repository.name,
manifest.digest,
))
try:
(report, state) = self._secscan_api.index(manifest, layers)
except InvalidContentSent as ex:
mark_manifest_unsupported(manifest)
logger.exception(
"Failed to perform indexing, invalid content sent")
continue
except APIRequestFailure as ex:
logger.exception(
"Failed to perform indexing, security scanner API error")
continue
if report["state"] == IndexReportState.Index_Finished:
index_status = IndexStatus.COMPLETED
elif report["state"] == IndexReportState.Index_Error:
index_status = IndexStatus.FAILED
else:
# Unknown state don't save anything
continue
with db_transaction():
ManifestSecurityStatus.delete().where(
ManifestSecurityStatus.manifest == candidate).execute()
ManifestSecurityStatus.create(
manifest=candidate,
repository=candidate.repository,
error_json=report["err"],
index_status=index_status,
indexer_hash=state,
indexer_version=IndexerVersion.V4,
metadata_json={},
)
def manifest_security_backfill_status():
manifest_count = Manifest.select().count()
mss_count = ManifestSecurityStatus.select().count()
return jsonify({"backfill_percent": mss_count / float(manifest_count)})
def perform_indexing(self, start_token=None):
whitelisted_namespaces = self.app.config.get(
"SECURITY_SCANNER_V4_NAMESPACE_WHITELIST", [])
try:
indexer_state = self._secscan_api.state()
except APIRequestFailure:
return None
def eligible_manifests(base_query):
return (base_query.join(Repository).join(User).where(
User.username << whitelisted_namespaces))
min_id = (start_token.min_id if start_token is not None else
Manifest.select(fn.Min(Manifest.id)).scalar())
max_id = Manifest.select(fn.Max(Manifest.id)).scalar()
if max_id is None or min_id is None or min_id > max_id:
return None
reindex_threshold = lambda: datetime.utcnow() - timedelta(
seconds=self.app.config.get("SECURITY_SCANNER_V4_REINDEX_THRESHOLD"
))
# TODO(alecmerdler): Filter out any `Manifests` that are still being uploaded
def not_indexed_query():
return (eligible_manifests(
Manifest.select()).switch(Manifest).join(
ManifestSecurityStatus,
JOIN.LEFT_OUTER).where(ManifestSecurityStatus.id >> None))
def index_error_query():
return (eligible_manifests(Manifest.select()).switch(
Manifest).join(ManifestSecurityStatus).where(
ManifestSecurityStatus.index_status == IndexStatus.FAILED,
ManifestSecurityStatus.last_indexed < reindex_threshold(),
))
def needs_reindexing_query(indexer_hash):
return (eligible_manifests(Manifest.select()).switch(
Manifest).join(ManifestSecurityStatus).where(
ManifestSecurityStatus.indexer_hash != indexer_hash,
ManifestSecurityStatus.last_indexed < reindex_threshold(),
))
# 4^log10(total) gives us a scalable batch size into the billions.
batch_size = int(4**log10(max(10, max_id - min_id)))
iterator = itertools.chain(
yield_random_entries(
not_indexed_query,
Manifest.id,
batch_size,
max_id,
min_id,
),
yield_random_entries(
index_error_query,
Manifest.id,
batch_size,
max_id,
min_id,
),
yield_random_entries(
lambda: needs_reindexing_query(indexer_state.get("state", "")),
Manifest.id,
batch_size,
max_id,
min_id,
),
)
for candidate, abt, num_remaining in iterator:
manifest = ManifestDataType.for_manifest(candidate, None)
layers = registry_model.list_manifest_layers(
manifest, self.storage, True)
logger.debug("Indexing %s/%s@%s" %
(candidate.repository.namespace_user,
candidate.repository.name, manifest.digest))
try:
(report, state) = self._secscan_api.index(manifest, layers)
except APIRequestFailure:
logger.exception(
"Failed to perform indexing, security scanner API error")
return None
with db_transaction():
ManifestSecurityStatus.delete().where(
ManifestSecurityStatus.manifest == candidate).execute()
ManifestSecurityStatus.create(
manifest=candidate,
repository=candidate.repository,
error_json=report["err"],
index_status=(IndexStatus.FAILED if report["state"]
== IndexReportState.Index_Error else
IndexStatus.COMPLETED),
indexer_hash=state,
indexer_version=IndexerVersion.V4,
metadata_json={},
)
return ScanToken(max_id + 1)