def test_perform_indexing_failed(initialized_db, set_secscan_config):
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.return_value = {"state": "abc"}
secscan._secscan_api.index.return_value = (
{
"err": None,
"state": IndexReportState.Index_Finished
},
"abc",
)
for manifest in Manifest.select():
ManifestSecurityStatus.create(
manifest=manifest,
repository=manifest.repository,
error_json={},
index_status=IndexStatus.FAILED,
indexer_hash="abc",
indexer_version=IndexerVersion.V4,
last_indexed=datetime.utcnow() - timedelta(
seconds=app.config["SECURITY_SCANNER_V4_REINDEX_THRESHOLD"] +
60),
metadata_json={},
)
secscan.perform_indexing()
assert ManifestSecurityStatus.select().count() == Manifest.select().count()
for mss in ManifestSecurityStatus.select():
assert mss.index_status == IndexStatus.COMPLETED
def test_perform_indexing_needs_reindexing_skip_unsupported(
initialized_db, set_secscan_config):
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.return_value = {"state": "new hash"}
secscan._secscan_api.index.return_value = (
{
"err": None,
"state": IndexReportState.Index_Finished
},
"new hash",
)
for manifest in Manifest.select():
ManifestSecurityStatus.create(
manifest=manifest,
repository=manifest.repository,
error_json={},
index_status=IndexStatus.MANIFEST_UNSUPPORTED,
indexer_hash="old hash",
indexer_version=IndexerVersion.V4,
last_indexed=datetime.utcnow() - timedelta(
seconds=app.config["SECURITY_SCANNER_V4_REINDEX_THRESHOLD"] +
60),
metadata_json={},
)
secscan.perform_indexing()
# Since this manifest should not be scanned, the old hash should remain
assert ManifestSecurityStatus.select().count() == Manifest.select().count()
for mss in ManifestSecurityStatus.select():
assert mss.indexer_hash == "old hash"
def test_perform_indexing_api_request_failure_index(initialized_db):
app.config["SECURITY_SCANNER_V4_NAMESPACE_WHITELIST"] = ["devtable"]
expected_manifests = (
Manifest.select(fn.Max(Manifest.id))
.join(Repository)
.join(User)
.where(User.username == "devtable")
)
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.return_value = "abc"
secscan._secscan_api.index.side_effect = APIRequestFailure()
next_token = secscan.perform_indexing()
assert next_token is None
assert ManifestSecurityStatus.select().count() == 0
# Set security scanner to return good results and attempt indexing again
secscan._secscan_api.index.side_effect = None
secscan._secscan_api.index.return_value = (
{"err": None, "state": IndexReportState.Index_Finished},
"abc",
)
next_token = secscan.perform_indexing()
assert next_token.min_id == expected_manifests.scalar() + 1
assert ManifestSecurityStatus.select().count() == expected_manifests.count()
def test_perform_indexing_api_request_failure_index(initialized_db,
set_secscan_config):
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.return_value = {"state": "abc"}
secscan._secscan_api.index.side_effect = APIRequestFailure()
next_token = secscan.perform_indexing()
assert next_token is None
assert ManifestSecurityStatus.select().count() == 0
# Set security scanner to return good results and attempt indexing again
secscan._secscan_api.index.side_effect = None
secscan._secscan_api.index.return_value = (
{
"err": None,
"state": IndexReportState.Index_Finished
},
"abc",
)
next_token = secscan.perform_indexing()
assert next_token.min_id == Manifest.select(fn.Max(
Manifest.id)).scalar() + 1
assert ManifestSecurityStatus.select().count() == Manifest.select(
fn.Max(Manifest.id)).count()
def test_perform_indexing_needs_reindexing(initialized_db):
app.config["SECURITY_SCANNER_V4_NAMESPACE_WHITELIST"] = ["devtable"]
expected_manifests = (
Manifest.select().join(Repository).join(User).where(User.username == "devtable")
)
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.return_value = "xyz"
secscan._secscan_api.index.return_value = (
{"err": None, "state": IndexReportState.Index_Finished},
"xyz",
)
for manifest in expected_manifests:
ManifestSecurityStatus.create(
manifest=manifest,
repository=manifest.repository,
error_json={},
index_status=IndexStatus.COMPLETED,
indexer_hash="abc",
indexer_version=IndexerVersion.V4,
metadata_json={},
)
secscan.perform_indexing()
assert ManifestSecurityStatus.select().count() == expected_manifests.count()
for mss in ManifestSecurityStatus.select():
assert mss.indexer_hash == "xyz"
def test_perform_indexing_whitelist(initialized_db, set_secscan_config):
app.config["SECURITY_SCANNER_V4_NAMESPACE_WHITELIST"] = ["devtable"]
expected_manifests = (Manifest.select().join(Repository).join(User).where(
User.username == "devtable"))
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.return_value = {"state": "abc"}
secscan._secscan_api.index.return_value = (
{
"err": None,
"state": IndexReportState.Index_Finished
},
"abc",
)
next_token = secscan.perform_indexing()
assert secscan._secscan_api.index.call_count == expected_manifests.count()
for mss in ManifestSecurityStatus.select():
assert mss.repository.namespace_user.username == "devtable"
assert ManifestSecurityStatus.select().count() == expected_manifests.count(
)
assert (
Manifest.get_by_id(next_token.min_id -
1).repository.namespace_user.username == "devtable")
def test_perform_indexing_failed_within_reindex_threshold(
initialized_db, set_secscan_config):
app.config["SECURITY_SCANNER_V4_REINDEX_THRESHOLD"] = 300
expected_manifests = (Manifest.select().join(Repository).join(User).where(
User.username == "devtable"))
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.return_value = {"state": "abc"}
secscan._secscan_api.index.return_value = (
{
"err": None,
"state": IndexReportState.Index_Finished
},
"abc",
)
for manifest in expected_manifests:
ManifestSecurityStatus.create(
manifest=manifest,
repository=manifest.repository,
error_json={},
index_status=IndexStatus.FAILED,
indexer_hash="abc",
indexer_version=IndexerVersion.V4,
metadata_json={},
)
secscan.perform_indexing()
assert ManifestSecurityStatus.select().count() == expected_manifests.count(
)
for mss in ManifestSecurityStatus.select():
assert mss.index_status == IndexStatus.FAILED
def test_perform_indexing_needs_reindexing_within_reindex_threshold(
initialized_db, set_secscan_config):
app.config["SECURITY_SCANNER_V4_REINDEX_THRESHOLD"] = 300
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.return_value = {"state": "xyz"}
secscan._secscan_api.index.return_value = (
{
"err": None,
"state": IndexReportState.Index_Finished
},
"xyz",
)
for manifest in Manifest.select():
ManifestSecurityStatus.create(
manifest=manifest,
repository=manifest.repository,
error_json={},
index_status=IndexStatus.COMPLETED,
indexer_hash="abc",
indexer_version=IndexerVersion.V4,
metadata_json={},
)
secscan.perform_indexing()
assert ManifestSecurityStatus.select().count() == Manifest.select().count()
for mss in ManifestSecurityStatus.select():
assert mss.indexer_hash == "abc"
def test_perform_indexing_invalid_manifest(initialized_db, set_secscan_config):
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
# Delete all ManifestBlob rows to cause the manifests to be invalid.
ManifestBlob.delete().execute()
secscan.perform_indexing()
assert ManifestSecurityStatus.select().count() == Manifest.select().count()
for mss in ManifestSecurityStatus.select():
assert mss.index_status == IndexStatus.MANIFEST_UNSUPPORTED
def test_perform_indexing_api_request_index_error_response(initialized_db, set_secscan_config):
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.return_value = {"state": "xyz"}
secscan._secscan_api.index.return_value = (
{"err": "something", "state": IndexReportState.Index_Error},
"xyz",
)
next_token = secscan.perform_indexing()
assert next_token.min_id == Manifest.select(fn.Max(Manifest.id)).scalar() + 1
assert ManifestSecurityStatus.select().count() == Manifest.select(fn.Max(Manifest.id)).count()
for mss in ManifestSecurityStatus.select():
assert mss.index_status == IndexStatus.FAILED
def test_perform_indexing_manifest_list(initialized_db, set_secscan_config):
repository_ref = registry_model.lookup_repository("devtable", "simple")
tag = registry_model.get_repo_tag(repository_ref, "latest")
manifest = registry_model.get_manifest_for_tag(tag)
Manifest.update(media_type=MediaType.get(
name=DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE)).execute()
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan.perform_indexing()
assert ManifestSecurityStatus.select().count() == Manifest.select().count()
for mss in ManifestSecurityStatus.select():
assert mss.index_status == IndexStatus.MANIFEST_UNSUPPORTED
def test_load_security_information_api_request_failure(initialized_db,
set_secscan_config):
repository_ref = registry_model.lookup_repository("devtable", "simple")
tag = registry_model.get_repo_tag(repository_ref,
"latest",
include_legacy_image=True)
manifest = registry_model.get_manifest_for_tag(tag,
backfill_if_necessary=True)
mss = ManifestSecurityStatus.create(
manifest=manifest._db_id,
repository=repository_ref._db_id,
error_json={},
index_status=IndexStatus.COMPLETED,
indexer_hash="abc",
indexer_version=IndexerVersion.V4,
metadata_json={},
)
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.vulnerability_report.side_effect = APIRequestFailure()
assert secscan.load_security_information(
manifest).status == ScanLookupStatus.COULD_NOT_LOAD
assert not ManifestSecurityStatus.select().where(
ManifestSecurityStatus.id == mss.id).exists()
def test_perform_indexing_whitelist(initialized_db, set_secscan_config):
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.return_value = {"state": "abc"}
secscan._secscan_api.index.return_value = (
{"err": None, "state": IndexReportState.Index_Finished},
"abc",
)
next_token = secscan.perform_indexing()
assert next_token.min_id == Manifest.select(fn.Max(Manifest.id)).scalar() + 1
assert secscan._secscan_api.index.call_count == Manifest.select().count()
assert ManifestSecurityStatus.select().count() == Manifest.select().count()
for mss in ManifestSecurityStatus.select():
assert mss.index_status == IndexStatus.COMPLETED
def test_perform_indexing_api_request_failure_state(initialized_db, set_secscan_config):
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.side_effect = APIRequestFailure()
next_token = secscan.perform_indexing()
assert next_token is None
assert ManifestSecurityStatus.select().count() == 0
def manifest_security_backfill_status():
manifest_count = Manifest.select().count()
mss_count = ManifestSecurityStatus.select().count()
if manifest_count == 0:
percent = 1.0
else:
percent = mss_count / float(manifest_count)
return jsonify({"backfill_percent": round(percent * 100, 2)})
def test_perform_indexing_api_request_failure_state(initialized_db):
app.config["SECURITY_SCANNER_V4_NAMESPACE_WHITELIST"] = ["devtable"]
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.side_effect = APIRequestFailure()
next_token = secscan.perform_indexing()
assert next_token is None
assert ManifestSecurityStatus.select().count() == 0
def test_perform_indexing_api_request_non_finished_state(initialized_db, set_secscan_config):
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.return_value = {"state": "xyz"}
secscan._secscan_api.index.return_value = (
{"err": "something", "state": "ScanLayers"},
"xyz",
)
next_token = secscan.perform_indexing()
assert next_token and next_token.min_id == Manifest.select(fn.Max(Manifest.id)).scalar() + 1
assert ManifestSecurityStatus.select().count() == 0
def test_perform_indexing_empty_whitelist(initialized_db):
app.config["SECURITY_SCANNER_V4_NAMESPACE_WHITELIST"] = []
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.return_value = "abc"
secscan._secscan_api.index.return_value = (
{"err": None, "state": IndexReportState.Index_Finished},
"abc",
)
next_token = secscan.perform_indexing()
assert secscan._secscan_api.index.call_count == 0
assert ManifestSecurityStatus.select().count() == 0
assert next_token.min_id == Manifest.select(fn.Max(Manifest.id)).scalar() + 1
def purge_repository(repo, force=False):
"""
Completely delete all traces of the repository.
Will return True upon complete success, and False upon partial or total failure. Garbage
collection is incremental and repeatable, so this return value does not need to be checked or
responded to.
"""
assert repo.state == RepositoryState.MARKED_FOR_DELETION or force
# Update the repo state to ensure nothing else is written to it.
repo.state = RepositoryState.MARKED_FOR_DELETION
repo.save()
# Delete the repository of all Appr-referenced entries.
# Note that new-model Tag's must be deleted in *two* passes, as they can reference parent tags,
# and MySQL is... particular... about such relationships when deleting.
if repo.kind.name == "application":
fst_pass = (ApprTag.delete().where(
ApprTag.repository == repo,
~(ApprTag.linked_tag >> None)).execute())
snd_pass = ApprTag.delete().where(ApprTag.repository == repo).execute()
gc_table_rows_deleted.labels(table="ApprTag").inc(fst_pass + snd_pass)
else:
# GC to remove the images and storage.
_purge_repository_contents(repo)
# Ensure there are no additional tags, manifests, images or blobs in the repository.
assert ApprTag.select().where(ApprTag.repository == repo).count() == 0
assert Tag.select().where(Tag.repository == repo).count() == 0
assert RepositoryTag.select().where(
RepositoryTag.repository == repo).count() == 0
assert Manifest.select().where(Manifest.repository == repo).count() == 0
assert ManifestBlob.select().where(
ManifestBlob.repository == repo).count() == 0
assert UploadedBlob.select().where(
UploadedBlob.repository == repo).count() == 0
assert (ManifestSecurityStatus.select().where(
ManifestSecurityStatus.repository == repo).count() == 0)
assert Image.select().where(Image.repository == repo).count() == 0
# Delete any repository build triggers, builds, and any other large-ish reference tables for
# the repository.
_chunk_delete_all(repo, RepositoryPermission, force=force)
_chunk_delete_all(repo, RepositoryBuild, force=force)
_chunk_delete_all(repo, RepositoryBuildTrigger, force=force)
_chunk_delete_all(repo, RepositoryActionCount, force=force)
_chunk_delete_all(repo, Star, force=force)
_chunk_delete_all(repo, AccessToken, force=force)
_chunk_delete_all(repo, RepositoryNotification, force=force)
_chunk_delete_all(repo, BlobUpload, force=force)
_chunk_delete_all(repo, RepoMirrorConfig, force=force)
_chunk_delete_all(repo, RepositoryAuthorizedEmail, force=force)
# Delete any marker rows for the repository.
DeletedRepository.delete().where(
DeletedRepository.repository == repo).execute()
# Delete the rest of the repository metadata.
try:
# Make sure the repository still exists.
fetched = Repository.get(id=repo.id)
except Repository.DoesNotExist:
return False
try:
fetched.delete_instance(recursive=True,
delete_nullable=False,
force=force)
gc_repos_purged.inc()
return True
except IntegrityError:
return False
def manifest_security_backfill_status():
manifest_count = Manifest.select().count()
mss_count = ManifestSecurityStatus.select().count()
return jsonify({"backfill_percent": mss_count / float(manifest_count)})