def test_dlc_major_version_label(image, region):
"""
Test to ensure that all DLC images have the LABEL "dlc_major_version"
:param image: <str> Image URI
:param region: <str> region where ECR repository holding the image resides
:return:
"""
ecr_client = boto3.client("ecr", region_name=region)
image_repository, image_tag = test_utils.get_repository_and_tag_from_image_uri(image)
# Using "acceptedMediaTypes" on the batch_get_image request allows the returned image information to
# provide the ECR Image Manifest in the specific format that we need, so that the image LABELS can be found
# on the manifest. The default format does not return the image LABELs.
response = ecr_client.batch_get_image(
repositoryName=image_repository,
imageIds=[{"imageTag": image_tag}],
acceptedMediaTypes=["application/vnd.docker.distribution.manifest.v1+json"],
)
if not response.get("images"):
raise KeyError(
f"Failed to get images through ecr_client.batch_get_image response for image {image_repository}:{image_tag}"
)
elif not response["images"][0].get("imageManifest"):
raise KeyError(f"imageManifest not found in ecr_client.batch_get_image response:\n{response['images']}")
manifest_str = response["images"][0]["imageManifest"]
# manifest_str is a json-format string
manifest = json.loads(manifest_str)
image_metadata = json.loads(manifest["history"][0]["v1Compatibility"])
major_version = image_metadata["config"]["Labels"].get("dlc_major_version", None)
assert major_version, f"{image} has no LABEL named 'dlc_major_version'. Please insert label."
test_utils.LOGGER.info(f"{image} has 'dlc_major_version' = {major_version}")
def test_framework_version_cpu(image):
"""
Check that the framework version in the image tag is the same as the one on a running container.
This function tests CPU, EIA, and Neuron images.
:param image: ECR image URI
"""
if "gpu" in image:
pytest.skip("GPU images will have their framework version tested in test_framework_and_cuda_version_gpu")
image_repo_name, _ = get_repository_and_tag_from_image_uri(image)
if re.fullmatch(r"(pr-|beta-|nightly-)?tensorflow-inference(-eia)?", image_repo_name):
pytest.skip(msg="TF inference for CPU/GPU/EIA does not have core tensorflow installed")
tested_framework, tag_framework_version = get_framework_and_version_from_tag(image)
# Framework name may include huggingface
tested_framework = tested_framework.lstrip("huggingface_")
# Module name is torch
if tested_framework == "pytorch":
tested_framework = "torch"
ctx = Context()
container_name = get_container_name("framework-version", image)
start_container(container_name, image, ctx)
output = run_cmd_on_container(
container_name, ctx, f"import {tested_framework}; print({tested_framework}.__version__)", executable="python"
)
if is_canary_context():
assert tag_framework_version in output.stdout.strip()
else:
assert tag_framework_version == output.stdout.strip()
def _run_instance_role_disabled(image_uri, ec2_client, ec2_instance,
ec2_connection):
expected_tag_key = "aws-dlc-autogenerated-tag-do-not-delete"
ec2_instance_id, _ = ec2_instance
account_id = test_utils.get_account_id_from_image_uri(image_uri)
image_region = test_utils.get_region_from_image_uri(image_uri)
repo_name, image_tag = test_utils.get_repository_and_tag_from_image_uri(
image_uri)
framework, _ = test_utils.get_framework_and_version_from_tag(image_uri)
job_type = test_utils.get_job_type_from_image(image_uri)
processor = test_utils.get_processor_from_image_uri(image_uri)
container_name = f"{repo_name}-telemetry_bad_instance_role-ec2"
docker_cmd = "nvidia-docker" if processor == "gpu" else "docker"
test_utils.login_to_ecr_registry(ec2_connection, account_id, image_region)
ec2_connection.run(f"{docker_cmd} pull -q {image_uri}")
preexisting_ec2_instance_tags = ec2_utils.get_ec2_instance_tags(
ec2_instance_id, ec2_client=ec2_client)
if expected_tag_key in preexisting_ec2_instance_tags:
ec2_client.remove_tags(Resources=[ec2_instance_id],
Tags=[{
"Key": expected_tag_key
}])
# Disable access to EC2 instance metadata
ec2_connection.run(f"sudo route add -host 169.254.169.254 reject")
if "tensorflow" in framework and job_type == "inference":
model_name = "saved_model_half_plus_two"
model_base_path = test_utils.get_tensorflow_model_base_path(image_uri)
env_vars_list = test_utils.get_tensorflow_inference_environment_variables(
model_name, model_base_path)
env_vars = " ".join([
f"-e {entry['name']}={entry['value']}" for entry in env_vars_list
])
inference_command = get_tensorflow_inference_command_tf27_above(
image_uri, model_name)
ec2_connection.run(
f"{docker_cmd} run {env_vars} --name {container_name} -id {image_uri} {inference_command}"
)
time.sleep(5)
else:
framework_to_import = framework.replace("huggingface_", "")
framework_to_import = "torch" if framework_to_import == "pytorch" else framework_to_import
ec2_connection.run(
f"{docker_cmd} run --name {container_name} -id {image_uri} bash")
output = ec2_connection.run(
f"{docker_cmd} exec -i {container_name} python -c 'import {framework_to_import}; import time; time.sleep(5)'",
warn=True)
assert output.ok, f"'import {framework_to_import}' fails when credentials not configured"
ec2_instance_tags = ec2_utils.get_ec2_instance_tags(ec2_instance_id,
ec2_client=ec2_client)
assert expected_tag_key not in ec2_instance_tags, (
f"{expected_tag_key} was applied as an instance tag."
"EC2 create_tags went through even though it should not have")
def test_framework_version_cpu(image):
"""
Check that the framework version in the image tag is the same as the one on a running container.
This function tests CPU, EIA, and Neuron images.
:param image: ECR image URI
"""
if "gpu" in image:
pytest.skip(
"GPU images will have their framework version tested in test_framework_and_cuda_version_gpu"
)
image_repo_name, _ = get_repository_and_tag_from_image_uri(image)
if re.fullmatch(
r"(pr-|beta-|nightly-)?tensorflow-inference(-eia|-graviton)?",
image_repo_name):
pytest.skip(
msg=
"TF inference for CPU/GPU/EIA does not have core tensorflow installed"
)
tested_framework, tag_framework_version = get_framework_and_version_from_tag(
image)
# Framework name may include huggingface
if tested_framework.startswith('huggingface_'):
tested_framework = tested_framework[len("huggingface_"):]
# Module name is torch
if tested_framework == "pytorch":
tested_framework = "torch"
elif tested_framework == "autogluon":
tested_framework = "autogluon.core"
ctx = Context()
container_name = get_container_name("framework-version", image)
start_container(container_name, image, ctx)
output = run_cmd_on_container(
container_name,
ctx,
f"import {tested_framework}; print({tested_framework}.__version__)",
executable="python")
if is_canary_context():
assert tag_framework_version in output.stdout.strip()
else:
if tested_framework == "autogluon.core":
assert output.stdout.strip().startswith(tag_framework_version)
elif tested_framework == "torch" and Version(
tag_framework_version) >= Version("1.10.0"):
torch_version_pattern = r"{torch_version}(\+cpu)".format(
torch_version=tag_framework_version)
assert re.fullmatch(
torch_version_pattern, output.stdout.strip()
), (f"torch.__version__ = {output.stdout.strip()} does not match {torch_version_pattern}\n"
f"Please specify framework version as X.Y.Z+cpu")
else:
if "neuron" in image:
assert tag_framework_version in output.stdout.strip()
else:
assert tag_framework_version == output.stdout.strip()
stop_and_remove_container(container_name, ctx)
def get_ecr_image_scan_severity_count(ecr_client, image_uri):
"""
Get ECR image scan findings
:param ecr_client: boto3 client for ECR
:param image_uri: image URI for image to be checked
:return: dict ECR Scan Findings
"""
repository, tag = get_repository_and_tag_from_image_uri(image_uri)
scan_info = ecr_client.describe_image_scan_findings(repositoryName=repository, imageId={"imageTag": tag})
severity_counts = scan_info["imageScanFindings"]["findingSeverityCounts"]
return severity_counts
def get_ecr_image_scan_status(ecr_client, image_uri):
"""
Get status of an ECR image scan in progress
:param ecr_client: boto3 client for ECR
:param image_uri: image URI for image to be checked
:return: tuple<str, str> Scan Status, Status Description
"""
repository, tag = get_repository_and_tag_from_image_uri(image_uri)
image_info = ecr_client.describe_images(repositoryName=repository, imageIds=[{"imageTag": tag}])["imageDetails"][0]
if "imageScanStatus" not in image_info.keys():
return None, "Scan not started"
return image_info["imageScanStatus"]["status"], image_info["imageScanStatus"].get("description", "NO DESCRIPTION")
def get_ecr_image_scan_time(ecr_client, image_uri):
"""
Find timestamp of when ECR Scan was last run for a given image URI assuming that this repository makes this
information available to the account running this function.
:param ecr_client: boto3 client for ECR
:param image_uri: image URI for image to be checked
:return: datetime.datetime object with UTC time or None
"""
repository, tag = get_repository_and_tag_from_image_uri(image_uri)
try:
scan_info = ecr_client.describe_image_scan_findings(repositoryName=repository, imageId={"imageTag": tag})
except ecr_client.exceptions.ScanNotFoundException:
return None
return scan_info["imageScanFindings"]["imageScanCompletedAt"]
def _run_tag_success(image_uri, ec2_client, ec2_instance, ec2_connection):
expected_tag_key = "aws-dlc-autogenerated-tag-do-not-delete"
ec2_instance_id, _ = ec2_instance
account_id = test_utils.get_account_id_from_image_uri(image_uri)
image_region = test_utils.get_region_from_image_uri(image_uri)
repo_name, image_tag = test_utils.get_repository_and_tag_from_image_uri(
image_uri)
framework, _ = test_utils.get_framework_and_version_from_tag(image_uri)
job_type = test_utils.get_job_type_from_image(image_uri)
processor = test_utils.get_processor_from_image_uri(image_uri)
container_name = f"{repo_name}-telemetry_tag_instance_success-ec2"
docker_cmd = "nvidia-docker" if processor == "gpu" else "docker"
test_utils.login_to_ecr_registry(ec2_connection, account_id, image_region)
ec2_connection.run(f"{docker_cmd} pull -q {image_uri}")
preexisting_ec2_instance_tags = ec2_utils.get_ec2_instance_tags(
ec2_instance_id, ec2_client=ec2_client)
if expected_tag_key in preexisting_ec2_instance_tags:
ec2_client.remove_tags(Resources=[ec2_instance_id],
Tags=[{
"Key": expected_tag_key
}])
if framework == "tensorflow" and job_type == "inference":
env_vars_list = ecs_utils.get_ecs_tensorflow_environment_variables(
processor, "saved_model_half_plus_two")
env_vars = " ".join([
f"-e {entry['name']}={entry['value']}" for entry in env_vars_list
])
ec2_connection.run(
f"{docker_cmd} run {env_vars} --name {container_name} -id {image_uri}"
)
time.sleep(5)
else:
framework_to_import = framework.replace("huggingface_", "")
framework_to_import = "torch" if framework_to_import == "pytorch" else framework_to_import
ec2_connection.run(
f"{docker_cmd} run --name {container_name} -id {image_uri} bash")
output = ec2_connection.run(
f"{docker_cmd} exec -i {container_name} python -c 'import {framework_to_import}; import time; time.sleep(5)'",
warn=True)
ec2_instance_tags = ec2_utils.get_ec2_instance_tags(ec2_instance_id,
ec2_client=ec2_client)
assert expected_tag_key in ec2_instance_tags, f"{expected_tag_key} was not applied as an instance tag"
def start_ecr_image_scan(ecr_client, image_uri):
"""
Start ECR Scan for an image, and Warn if scan cannot be started
:param ecr_client: boto3 client for ECR
:param image_uri: image URI for image to be checked
"""
repository, tag = get_repository_and_tag_from_image_uri(image_uri)
try:
scan_info = ecr_client.start_image_scan(repositoryName=repository, imageId={"imageTag": tag})
except ecr_client.exceptions.LimitExceededException:
LOGGER.warning("Scan has already been run on this image in the last 24 hours.")
return
if scan_info["imageScanStatus"]["status"] == "FAILED":
raise ECRScanFailedError(f"ECR Scan failed and returned:\n{json.dumps(scan_info, indent=4)}")
return
def test_tf_serving_version_cpu(tensorflow_inference):
"""
For non-huggingface non-GPU TF inference images, check that the tag version matches the version of TF serving
in the container.
Huggingface includes MMS and core TF, hence the versioning scheme is based off of the underlying tensorflow
framework version, rather than the TF serving version.
GPU inference images will be tested along side `test_framework_and_cuda_version_gpu` in order to be judicious
about GPU resources. This test can run directly on the host, and thus does not require additional resources
to be spun up.
@param tensorflow_inference: ECR image URI
"""
# Set local variable to clarify contents of fixture
image = tensorflow_inference
if "gpu" in image:
pytest.skip(
"GPU images will have their framework version tested in test_framework_and_cuda_version_gpu"
)
if "neuron" in image:
pytest.skip(
"Neuron images will have their framework version tested in test_framework_and_neuron_sdk_version"
)
_, tag_framework_version = get_framework_and_version_from_tag(image)
image_repo_name, _ = get_repository_and_tag_from_image_uri(image)
if re.fullmatch(r"(pr-|beta-|nightly-)?tensorflow-inference",
image_repo_name) and Version(
tag_framework_version) == Version("2.6.3"):
pytest.skip(
"Skipping this test for TF 2.6.3 inference as the v2.6.3 version is already on production"
)
ctx = Context()
container_name = get_container_name("tf-serving-version", image)
start_container(container_name, image, ctx)
output = run_cmd_on_container(container_name,
ctx,
"tensorflow_model_server --version",
executable="bash")
assert re.match(rf"TensorFlow ModelServer: {tag_framework_version}(\D+)?", output.stdout), \
f"Cannot find model server version {tag_framework_version} in {output.stdout}"
stop_and_remove_container(container_name, ctx)
def get_ecr_image_scan_results(ecr_client, image_uri, minimum_vulnerability="HIGH"):
"""
Get list of vulnerabilities from ECR image scan results
:param ecr_client:
:param image_uri:
:param minimum_vulnerability: str representing minimum vulnerability level to report in results
:return: list<dict> Scan results
"""
repository, tag = get_repository_and_tag_from_image_uri(image_uri)
scan_info = ecr_client.describe_image_scan_findings(repositoryName=repository, imageId={"imageTag": tag})
scan_findings = [
finding
for finding in scan_info["imageScanFindings"]["findings"]
if CVESeverity[finding["severity"]] >= CVESeverity[minimum_vulnerability]
]
return scan_findings
def get_new_image_uri_for_uploading_upgraded_image_to_ecr(image):
"""
Returns the new image uri for the image being tested. After running the apt commands, the
new image will be uploaded to the ECR based on the new image uri.
:param image: str
:param new_image_uri: str
"""
new_repository_name = test_utils.UPGRADE_ECR_REPO_NAME
region = os.getenv("REGION", test_utils.DEFAULT_REGION)
sts_client = boto3.client("sts", region_name=region)
account_id = sts_client.get_caller_identity().get("Account")
registry = ecr_utils.get_ecr_registry(account_id, region)
original_image_repository, original_image_tag = test_utils.get_repository_and_tag_from_image_uri(
image)
upgraded_image_tag = f"{original_image_repository}-{original_image_tag}-upgraded"
new_image_uri = f"{registry}/{new_repository_name}:{upgraded_image_tag}"
return new_image_uri
def test_framework_and_cuda_version_gpu(gpu, ec2_connection):
"""
Check that the framework and cuda version in the image tag is the same as the one on a running container.
:param gpu: ECR image URI with "gpu" in the name
:param ec2_connection: fixture to establish connection with an ec2 instance
"""
image = gpu
tested_framework, tag_framework_version = get_framework_and_version_from_tag(
image)
image_repo_name, _ = get_repository_and_tag_from_image_uri(image)
# Framework Version Check #
# For tf inference containers, check TF model server version
if re.fullmatch(
r"(pr-|beta-|nightly-)?tensorflow-inference(-eia|-graviton)?",
image_repo_name):
cmd = f"tensorflow_model_server --version"
output = ec2.execute_ec2_training_test(ec2_connection,
image,
cmd,
executable="bash")
assert (re.match(
rf"TensorFlow Model Server: {tag_framework_version}(\D+)?",
output.stdout
), f"Cannot find model server version {tag_framework_version} in {output.stdout}"
)
else:
# Framework name may include huggingface
if tested_framework.startswith('huggingface_'):
tested_framework = tested_framework[len("huggingface_"):]
# Module name is "torch"
if tested_framework == "pytorch":
tested_framework = "torch"
elif tested_framework == "autogluon":
tested_framework = "autogluon.core"
cmd = f"import {tested_framework}; print({tested_framework}.__version__)"
output = ec2.execute_ec2_training_test(ec2_connection,
image,
cmd,
executable="python")
if is_canary_context():
assert tag_framework_version in output.stdout.strip()
else:
if tested_framework == "autogluon.core":
version_to_check = "0.3.1" if tag_framework_version == "0.3.2" else tag_framework_version
assert output.stdout.strip().startswith(version_to_check)
elif tested_framework == "torch" and Version(
tag_framework_version) >= Version("1.10.0"):
torch_version_pattern = r"{torch_version}(\+cu\d+)".format(
torch_version=tag_framework_version)
assert re.fullmatch(
torch_version_pattern, output.stdout.strip()
), (f"torch.__version__ = {output.stdout.strip()} does not match {torch_version_pattern}\n"
f"Please specify framework version as X.Y.Z+cuXXX")
else:
assert tag_framework_version == output.stdout.strip()
# CUDA Version Check #
cuda_version = re.search(r"-cu(\d+)-", image).group(1)
# MXNet inference/HF tensorflow inference and Autogluon containers do not currently have nvcc in /usr/local/cuda/bin, so check symlink
if "mxnet-inference" in image or "autogluon" in image or "huggingface-tensorflow-inference" in image:
cuda_cmd = "readlink /usr/local/cuda"
else:
cuda_cmd = "nvcc --version"
cuda_output = ec2.execute_ec2_training_test(
ec2_connection, image, cuda_cmd, container_name="cuda_version_test")
# Ensure that cuda version in tag is in the container
assert cuda_version in cuda_output.stdout.replace(".", "")
def test_framework_version_cpu(image):
"""
Check that the framework version in the image tag is the same as the one on a running container.
This function tests CPU, EIA images.
:param image: ECR image URI
"""
if "gpu" in image:
pytest.skip(
"GPU images will have their framework version tested in test_framework_and_cuda_version_gpu"
)
if "neuron" in image:
pytest.skip(
"Neuron images will have their framework version tested in test_framework_and_neuron_sdk_version"
)
image_repo_name, _ = get_repository_and_tag_from_image_uri(image)
if re.fullmatch(
r"(pr-|beta-|nightly-)?tensorflow-inference(-eia|-graviton)?",
image_repo_name):
pytest.skip(
"Non-gpu tensorflow-inference images will be tested in test_tf_serving_version_cpu."
)
tested_framework, tag_framework_version = get_framework_and_version_from_tag(
image)
# Framework name may include huggingface
if tested_framework.startswith('huggingface_'):
tested_framework = tested_framework[len("huggingface_"):]
# Module name is torch
if tested_framework == "pytorch":
tested_framework = "torch"
elif tested_framework == "autogluon":
tested_framework = "autogluon.core"
ctx = Context()
container_name = get_container_name("framework-version", image)
start_container(container_name, image, ctx)
output = run_cmd_on_container(
container_name,
ctx,
f"import {tested_framework}; print({tested_framework}.__version__)",
executable="python")
if is_canary_context():
assert tag_framework_version in output.stdout.strip()
else:
if tested_framework == "autogluon.core":
version_to_check = "0.3.1" if tag_framework_version == "0.3.2" else tag_framework_version
assert output.stdout.strip().startswith(version_to_check)
# Habana v1.2 binary does not follow the X.Y.Z+cpu naming convention
elif "habana" not in image_repo_name:
if tested_framework == "torch" and Version(
tag_framework_version) >= Version("1.10.0"):
torch_version_pattern = r"{torch_version}(\+cpu)".format(
torch_version=tag_framework_version)
assert re.fullmatch(
torch_version_pattern, output.stdout.strip()
), (f"torch.__version__ = {output.stdout.strip()} does not match {torch_version_pattern}\n"
f"Please specify framework version as X.Y.Z+cpu")
else:
if "neuron" in image:
assert tag_framework_version in output.stdout.strip()
if all(_string in image
for _string in ["pytorch", "habana", "synapseai1.3.0"]):
# Habana Pytorch version looks like 1.10.0a0+gitb488e78 for SynapseAI1.3 PT1.10.1 images
pt_fw_version_pattern = r"(\d+(\.\d+){1,2}(-rc\d)?)((a0\+git\w{7}))"
pt_fw_version_match = re.fullmatch(pt_fw_version_pattern,
output.stdout.strip())
# This is desired for PT1.10.1 images
assert pt_fw_version_match.group(1) == "1.10.0"
else:
assert tag_framework_version == output.stdout.strip()
stop_and_remove_container(container_name, ctx)
def test_ecr_scan(image, ecr_client, sts_client, region):
"""
Run ECR Scan Tool on an image being tested, and raise Error if vulnerabilities found
1. Start Scan.
2. For 5 minutes (Run DescribeImages):
(We run this for 5 minutes because the Scan is expected to complete in about 2 minutes, though no
analysis has been performed on exactly how long the Scan takes for a DLC image. Therefore we also
have a 3 minute buffer beyond the expected amount of time taken.)
3.1. If imageScanStatus == COMPLETE: exit loop
3.2. If imageScanStatus == IN_PROGRESS or AttributeNotFound(imageScanStatus): continue loop
3.3. If imageScanStatus == FAILED: raise RuntimeError
4. If DescribeImages.imageScanStatus != COMPLETE: raise TimeOutError
5. assert imageScanFindingsSummary.findingSeverityCounts.HIGH/CRITICAL == 0
:param image: str Image URI for image to be tested
:param ecr_client: boto3 Client for ECR
:param sts_client: boto3 Client for STS
:param region: str Name of region where test is executed
"""
test_account_id = sts_client.get_caller_identity().get("Account")
image_account_id = get_account_id_from_image_uri(image)
image_region = get_region_from_image_uri(image)
image_repo_name, original_image_tag = get_repository_and_tag_from_image_uri(image)
additional_image_tags = get_all_the_tags_of_an_image_from_ecr(ecr_client, image)
if not is_image_available_locally(image):
LOGGER.info(f"Image {image} not available locally!! Pulling the image...")
login_to_ecr_registry(Context(), image_account_id, image_region)
run(f"docker pull {image}")
if not is_image_available_locally(image):
raise RuntimeError("Image shown as not available even after pulling")
for additional_tag in additional_image_tags:
image_uri_with_new_tag = image.replace(original_image_tag, additional_tag)
run(f"docker tag {image} {image_uri_with_new_tag}", hide=True)
if image_account_id != test_account_id:
original_image = image
target_image_repo_name = f"beta-{image_repo_name}"
for additional_tag in additional_image_tags:
image_uri_with_new_tag = original_image.replace(original_image_tag, additional_tag)
new_image_uri = ecr_utils.reupload_image_to_test_ecr(image_uri_with_new_tag, target_image_repo_name, region)
if image_uri_with_new_tag == original_image:
image = new_image_uri
minimum_sev_threshold = get_minimum_sev_threshold_level(image)
LOGGER.info(f"Severity threshold level is {minimum_sev_threshold}")
run_scan(ecr_client, image)
scan_results = ecr_utils.get_ecr_image_scan_results(ecr_client, image, minimum_vulnerability=minimum_sev_threshold)
scan_results = ecr_utils.populate_ecr_scan_with_web_scraper_results(image, scan_results)
ecr_image_vulnerability_list = ScanVulnerabilityList(minimum_severity=CVESeverity[minimum_sev_threshold])
ecr_image_vulnerability_list.construct_allowlist_from_ecr_scan_result(scan_results)
remaining_vulnerabilities = ecr_image_vulnerability_list
if not is_image_covered_by_allowlist_feature(image):
if is_canary_context():
pytest.skip("Skipping the test on the canary.")
common_ecr_scan_allowlist = ScanVulnerabilityList(minimum_severity=CVESeverity[minimum_sev_threshold])
common_ecr_scan_allowlist_path = os.path.join(
os.sep, get_repository_local_path(), "data", "common-ecr-scan-allowlist.json"
)
if os.path.exists(common_ecr_scan_allowlist_path):
common_ecr_scan_allowlist.construct_allowlist_from_file(common_ecr_scan_allowlist_path)
remaining_vulnerabilities = remaining_vulnerabilities - common_ecr_scan_allowlist
if remaining_vulnerabilities:
assert not remaining_vulnerabilities.vulnerability_list, (
f"The following vulnerabilities need to be fixed on {image}:\n"
f"{json.dumps(remaining_vulnerabilities.vulnerability_list, indent=4)}"
)
return
upgraded_image_vulnerability_list, image_scan_allowlist = fetch_other_vulnerability_lists(
image, ecr_client, minimum_sev_threshold
)
s3_bucket_name = ECR_SCAN_HELPER_BUCKET
## In case new vulnerabilities (fixable or non-fixable) are found, then conduct failure routine
newly_found_vulnerabilities = ecr_image_vulnerability_list - image_scan_allowlist
# In case there is no new vulnerability but the allowlist is outdated
vulnerabilities_that_can_be_fixed = image_scan_allowlist - upgraded_image_vulnerability_list
if newly_found_vulnerabilities or vulnerabilities_that_can_be_fixed:
failure_routine_summary = conduct_failure_routine(
image,
image_scan_allowlist,
ecr_image_vulnerability_list,
upgraded_image_vulnerability_list,
s3_bucket_name,
)
(
s3_filename_for_fixable_list,
s3_filename_for_non_fixable_list,
) = process_failure_routine_summary_and_store_data_in_s3(failure_routine_summary, s3_bucket_name)
prepend_message = "Found new vulnerabilities in image." if newly_found_vulnerabilities else "Allowlist is outdated."
display_message = prepend_message + " " + (
f"""Found {len(failure_routine_summary["fixable_vulnerabilities"])} fixable vulnerabilites """
f"""and {len(failure_routine_summary["non_fixable_vulnerabilities"])} non fixable vulnerabilites. """
f"""Refer to files s3://{s3_bucket_name}/{s3_filename_for_fixable_list}, s3://{s3_bucket_name}/{s3_filename_for_non_fixable_list}, """
f"""s3://{s3_bucket_name}/{failure_routine_summary["s3_filename_for_current_image_ecr_scan_list"]} and s3://{s3_bucket_name}/{failure_routine_summary["s3_filename_for_allowlist"]}."""
)
if is_canary_context():
LOGGER.error(display_message)
pytest.skip("Skipping the test failure on the canary.")
else:
raise RuntimeError(display_message)
