def run_smclarify_bias_metrics(
image_uri,
ec2_connection,
ec2_instance_type,
docker_executable="docker",
container_name="smclarify",
test_script=SMCLARIFY_SCRIPT,
):
container_test_local_dir = os.path.join("$HOME", "container_tests")
account_id = get_account_id_from_image_uri(image_uri)
region = get_region_from_image_uri(image_uri)
login_to_ecr_registry(ec2_connection, account_id, region)
ec2_connection.run(f"docker pull -q {image_uri}")
try:
ec2_connection.run(
f"{docker_executable} run --name {container_name} -v "
f"{container_test_local_dir}:{os.path.join(os.sep, 'test')} {image_uri} "
f"python {test_script}",
hide=True,
timeout=300,
)
except Exception as e:
debug_output = ec2_connection.run(f"docker logs {container_name}")
debug_stdout = debug_output.stdout
if "Test SMClarify Bias Metrics succeeded!" in debug_stdout:
LOGGER.warning(
f"SMClarify test succeeded, but there is an issue with fabric. "
f"Error:\n{e}\nTest output:\n{debug_stdout}"
)
return
raise SMClarifyTestFailure(
f"SMClarify test failed on {image_uri} on {ec2_instance_type}. Full output:\n{debug_stdout}"
) from e
def _run_instance_role_disabled(image_uri, ec2_client, ec2_instance,
ec2_connection):
expected_tag_key = "aws-dlc-autogenerated-tag-do-not-delete"
ec2_instance_id, _ = ec2_instance
account_id = test_utils.get_account_id_from_image_uri(image_uri)
image_region = test_utils.get_region_from_image_uri(image_uri)
repo_name, image_tag = test_utils.get_repository_and_tag_from_image_uri(
image_uri)
framework, _ = test_utils.get_framework_and_version_from_tag(image_uri)
job_type = test_utils.get_job_type_from_image(image_uri)
processor = test_utils.get_processor_from_image_uri(image_uri)
container_name = f"{repo_name}-telemetry_bad_instance_role-ec2"
docker_cmd = "nvidia-docker" if processor == "gpu" else "docker"
test_utils.login_to_ecr_registry(ec2_connection, account_id, image_region)
ec2_connection.run(f"{docker_cmd} pull -q {image_uri}")
preexisting_ec2_instance_tags = ec2_utils.get_ec2_instance_tags(
ec2_instance_id, ec2_client=ec2_client)
if expected_tag_key in preexisting_ec2_instance_tags:
ec2_client.remove_tags(Resources=[ec2_instance_id],
Tags=[{
"Key": expected_tag_key
}])
# Disable access to EC2 instance metadata
ec2_connection.run(f"sudo route add -host 169.254.169.254 reject")
if "tensorflow" in framework and job_type == "inference":
model_name = "saved_model_half_plus_two"
model_base_path = test_utils.get_tensorflow_model_base_path(image_uri)
env_vars_list = test_utils.get_tensorflow_inference_environment_variables(
model_name, model_base_path)
env_vars = " ".join([
f"-e {entry['name']}={entry['value']}" for entry in env_vars_list
])
inference_command = get_tensorflow_inference_command_tf27_above(
image_uri, model_name)
ec2_connection.run(
f"{docker_cmd} run {env_vars} --name {container_name} -id {image_uri} {inference_command}"
)
time.sleep(5)
else:
framework_to_import = framework.replace("huggingface_", "")
framework_to_import = "torch" if framework_to_import == "pytorch" else framework_to_import
ec2_connection.run(
f"{docker_cmd} run --name {container_name} -id {image_uri} bash")
output = ec2_connection.run(
f"{docker_cmd} exec -i {container_name} python -c 'import {framework_to_import}; import time; time.sleep(5)'",
warn=True)
assert output.ok, f"'import {framework_to_import}' fails when credentials not configured"
ec2_instance_tags = ec2_utils.get_ec2_instance_tags(ec2_instance_id,
ec2_client=ec2_client)
assert expected_tag_key not in ec2_instance_tags, (
f"{expected_tag_key} was applied as an instance tag."
"EC2 create_tags went through even though it should not have")
def test_is_ecr_scan_allowlist_outdated(image, ecr_client, sts_client, region):
"""
Run ECR Scan Tool on an image being tested, and test if the vulnerabilities in the allowlist for the image
are still valid, or if any vulnerabilities must be removed from the list.
:param image: str Image URI for image to be tested
:param ecr_client: boto3 Client for ECR
:param sts_client: boto3 Client for STS
:param region: str Name of region where test is executed
"""
test_account_id = sts_client.get_caller_identity().get("Account")
image_account_id = get_account_id_from_image_uri(image)
if image_account_id != test_account_id:
image_repo_uri, image_tag = image.split(":")
_, image_repo_name = image_repo_uri.split("/")
target_image_repo_name = f"beta-{image_repo_name}"
image = ecr_utils.reupload_image_to_test_ecr(image, target_image_repo_name, region)
run_scan(ecr_client, image)
scan_results = ecr_utils.get_ecr_image_scan_results(ecr_client, image, minimum_vulnerability=MINIMUM_SEV_THRESHOLD)
image_scan_allowlist = ScanVulnerabilityList(minimum_severity=CVESeverity[MINIMUM_SEV_THRESHOLD])
image_scan_allowlist_path = get_ecr_scan_allowlist_path(image)
if os.path.exists(image_scan_allowlist_path):
image_scan_allowlist.construct_allowlist_from_file(image_scan_allowlist_path)
ecr_image_vulnerability_list = ScanVulnerabilityList(minimum_severity=CVESeverity[MINIMUM_SEV_THRESHOLD])
ecr_image_vulnerability_list.construct_allowlist_from_ecr_scan_result(scan_results)
invalid_allowlist_vulnerabilities = image_scan_allowlist - ecr_image_vulnerability_list
assert not invalid_allowlist_vulnerabilities, (
f"The following vulnerabilities are no longer valid on {image}:\n"
f"{json.dumps(invalid_allowlist_vulnerabilities.vulnerability_list, indent=4)}"
)
def reupload_image_to_test_ecr(source_image_uri, target_image_repo_name,
target_region):
"""
Helper function to reupload an image owned by a another/same account to an ECR repo in this account to given region, so that
this account can freely run tests without permission issues.
:param source_image_uri: str Image URI for image to be tested
:param target_image_repo_name: str Target image ECR repo name
:param target_region: str Region where test is being run
:return: str New image URI for re-uploaded image
"""
ECR_PASSWORD_FILE_PATH = os.path.join(
"/tmp", f"{get_unique_name_from_tag(source_image_uri)}.txt")
sts_client = boto3.client("sts", region_name=target_region)
target_ecr_client = boto3.client("ecr", region_name=target_region)
target_account_id = sts_client.get_caller_identity().get("Account")
image_account_id = get_account_id_from_image_uri(source_image_uri)
image_region = get_region_from_image_uri(source_image_uri)
image_repo_uri, image_tag = source_image_uri.split(":")
_, image_repo_name = image_repo_uri.split("/")
if not ecr_repo_exists(target_ecr_client, target_image_repo_name):
raise ECRRepoDoesNotExist(
f"Repo named {target_image_repo_name} does not exist in {target_region} on the account {target_account_id}"
)
target_image_uri = (source_image_uri.replace(
image_region, target_region).replace(image_repo_name,
target_image_repo_name).replace(
image_account_id,
target_account_id))
client = boto3.client('ecr', region_name=image_region)
username, password = get_ecr_login_boto3(client, image_account_id,
image_region)
save_credentials_to_file(ECR_PASSWORD_FILE_PATH, password)
# using ctx.run throws error on codebuild "OSError: reading from stdin while output is captured".
# Also it throws more errors related to awscli if in_stream=False flag is added to ctx.run which needs more deep dive
subprocess.check_output(
f"cat {ECR_PASSWORD_FILE_PATH} | docker login -u {username} --password-stdin https://{image_account_id}.dkr.ecr.{image_region}.amazonaws.com && docker pull {source_image_uri}",
shell=True,
executable="/bin/bash")
subprocess.check_output(
f"docker tag {source_image_uri} {target_image_uri}",
shell=True,
executable="/bin/bash")
delete_file(ECR_PASSWORD_FILE_PATH)
username, password = get_ecr_login_boto3(target_ecr_client,
target_account_id, target_region)
save_credentials_to_file(ECR_PASSWORD_FILE_PATH, password)
subprocess.check_output(
f"cat {ECR_PASSWORD_FILE_PATH} | docker login -u {username} --password-stdin https://{target_account_id}.dkr.ecr.{target_region}.amazonaws.com && docker push {target_image_uri}",
shell=True,
executable="/bin/bash")
delete_file(ECR_PASSWORD_FILE_PATH)
return target_image_uri
def test_ecr_scan(image, ecr_client, sts_client, region):
"""
Run ECR Scan Tool on an image being tested, and raise Error if vulnerabilities found
1. Start Scan.
2. For 5 minutes (Run DescribeImages):
(We run this for 5 minutes because the Scan is expected to complete in about 2 minutes, though no
analysis has been performed on exactly how long the Scan takes for a DLC image. Therefore we also
have a 3 minute buffer beyond the expected amount of time taken.)
3.1. If imageScanStatus == COMPLETE: exit loop
3.2. If imageScanStatus == IN_PROGRESS or AttributeNotFound(imageScanStatus): continue loop
3.3. If imageScanStatus == FAILED: raise RuntimeError
4. If DescribeImages.imageScanStatus != COMPLETE: raise TimeOutError
5. assert imageScanFindingsSummary.findingSeverityCounts.HIGH/CRITICAL == 0
:param image: str Image URI for image to be tested
:param ecr_client: boto3 Client for ECR
:param sts_client: boto3 Client for STS
:param region: str Name of region where test is executed
"""
test_account_id = sts_client.get_caller_identity().get("Account")
image_account_id = get_account_id_from_image_uri(image)
if image_account_id != test_account_id:
image_repo_uri, image_tag = image.split(":")
_, image_repo_name = image_repo_uri.split("/")
target_image_repo_name = f"beta-{image_repo_name}"
image = ecr_utils.reupload_image_to_test_ecr(image, target_image_repo_name, region)
run_scan(ecr_client, image)
scan_results = ecr_utils.get_ecr_image_scan_results(ecr_client, image, minimum_vulnerability=MINIMUM_SEV_THRESHOLD)
ecr_image_vulnerability_list = ScanVulnerabilityList(minimum_severity=CVESeverity[MINIMUM_SEV_THRESHOLD])
ecr_image_vulnerability_list.construct_allowlist_from_ecr_scan_result(scan_results)
remaining_vulnerabilities = ecr_image_vulnerability_list
# TODO: Once this feature is enabled, remove "if" condition and second assertion statement
# TODO: Ensure this works on the canary tags before removing feature flag
if is_ecr_scan_allowlist_feature_enabled():
image_scan_allowlist = ScanVulnerabilityList(minimum_severity=CVESeverity[MINIMUM_SEV_THRESHOLD])
image_scan_allowlist_path = get_ecr_scan_allowlist_path(image)
if os.path.exists(image_scan_allowlist_path):
image_scan_allowlist.construct_allowlist_from_file(image_scan_allowlist_path)
remaining_vulnerabilities = ecr_image_vulnerability_list - image_scan_allowlist
assert not remaining_vulnerabilities, (
f"The following vulnerabilities need to be fixed on {image}:\n"
f"{json.dumps(remaining_vulnerabilities.vulnerability_list, indent=4)}"
)
return
assert not remaining_vulnerabilities.vulnerability_list, (
f"The following vulnerabilities need to be fixed on {image}:\n"
f"{json.dumps(remaining_vulnerabilities.vulnerability_list, indent=4)}"
)
def test_ecr_scan(image, ecr_client, sts_client, region):
"""
Run ECR Scan Tool on an image being tested, and raise Error if vulnerabilities found
1. Start Scan.
2. For 5 minutes (Run DescribeImages):
(We run this for 5 minutes because the Scan is expected to complete in about 2 minutes, though no
analysis has been performed on exactly how long the Scan takes for a DLC image. Therefore we also
have a 3 minute buffer beyond the expected amount of time taken.)
3.1. If imageScanStatus == COMPLETE: exit loop
3.2. If imageScanStatus == IN_PROGRESS or AttributeNotFound(imageScanStatus): continue loop
3.3. If imageScanStatus == FAILED: raise RuntimeError
4. If DescribeImages.imageScanStatus != COMPLETE: raise TimeOutError
5. assert imageScanFindingsSummary.findingSeverityCounts.HIGH/CRITICAL == 0
:param image: str Image URI for image to be tested
:param ecr_client: boto3 Client for ECR
:param sts_client: boto3 Client for STS
:param region: str Name of region where test is executed
"""
test_account_id = sts_client.get_caller_identity().get("Account")
image_account_id = get_account_id_from_image_uri(image)
if image_account_id != test_account_id:
image = _reupload_image_to_test_ecr(image, ecr_client, region,
test_account_id)
minimum_sev_threshold = "HIGH"
scan_status = None
start_time = time()
ecr_utils.start_ecr_image_scan(ecr_client, image)
while (time() - start_time) <= 600:
scan_status, scan_status_description = ecr_utils.get_ecr_image_scan_status(
ecr_client, image)
if scan_status == "FAILED" or scan_status not in [
None, "IN_PROGRESS", "COMPLETE"
]:
raise RuntimeError(scan_status_description)
if scan_status == "COMPLETE":
break
sleep(1)
if scan_status != "COMPLETE":
raise TimeoutError(
f"ECR Scan is still in {scan_status} state. Exiting.")
severity_counts = ecr_utils.get_ecr_image_scan_severity_count(
ecr_client, image)
scan_results = ecr_utils.get_ecr_image_scan_results(
ecr_client, image, minimum_vulnerability=minimum_sev_threshold)
assert all(
count == 0 for sev, count in severity_counts.items()
if CVESeverity[sev] >= CVESeverity[minimum_sev_threshold]), (
f"Found vulnerabilities in image {image}: {str(severity_counts)}\n"
f"Vulnerabilities: {json.dumps(scan_results, indent=4)}")
def _run_tag_success(image_uri, ec2_client, ec2_instance, ec2_connection):
expected_tag_key = "aws-dlc-autogenerated-tag-do-not-delete"
ec2_instance_id, _ = ec2_instance
account_id = test_utils.get_account_id_from_image_uri(image_uri)
image_region = test_utils.get_region_from_image_uri(image_uri)
repo_name, image_tag = test_utils.get_repository_and_tag_from_image_uri(
image_uri)
framework, _ = test_utils.get_framework_and_version_from_tag(image_uri)
job_type = test_utils.get_job_type_from_image(image_uri)
processor = test_utils.get_processor_from_image_uri(image_uri)
container_name = f"{repo_name}-telemetry_tag_instance_success-ec2"
docker_cmd = "nvidia-docker" if processor == "gpu" else "docker"
test_utils.login_to_ecr_registry(ec2_connection, account_id, image_region)
ec2_connection.run(f"{docker_cmd} pull -q {image_uri}")
preexisting_ec2_instance_tags = ec2_utils.get_ec2_instance_tags(
ec2_instance_id, ec2_client=ec2_client)
if expected_tag_key in preexisting_ec2_instance_tags:
ec2_client.remove_tags(Resources=[ec2_instance_id],
Tags=[{
"Key": expected_tag_key
}])
if framework == "tensorflow" and job_type == "inference":
env_vars_list = ecs_utils.get_ecs_tensorflow_environment_variables(
processor, "saved_model_half_plus_two")
env_vars = " ".join([
f"-e {entry['name']}={entry['value']}" for entry in env_vars_list
])
ec2_connection.run(
f"{docker_cmd} run {env_vars} --name {container_name} -id {image_uri}"
)
time.sleep(5)
else:
framework_to_import = framework.replace("huggingface_", "")
framework_to_import = "torch" if framework_to_import == "pytorch" else framework_to_import
ec2_connection.run(
f"{docker_cmd} run --name {container_name} -id {image_uri} bash")
output = ec2_connection.run(
f"{docker_cmd} exec -i {container_name} python -c 'import {framework_to_import}; import time; time.sleep(5)'",
warn=True)
ec2_instance_tags = ec2_utils.get_ec2_instance_tags(ec2_instance_id,
ec2_client=ec2_client)
assert expected_tag_key in ec2_instance_tags, f"{expected_tag_key} was not applied as an instance tag"
def _reupload_image_to_test_ecr(source_image_uri, test_ecr_client, test_region,
test_account_id):
"""
Helper function to reupload an image owned by a different account to an ECR repo in this account, so that
this account can freely run ECR Scan without permission issues.
:param source_image_uri: str Image URI for image to be tested
:param test_ecr_client: boto3.Client ECR client for account where test is being run
:param test_region: str Region where test is being run
:param test_account_id: str Account ID for account where test is being run
:return: str New image URI for re-uploaded image
"""
ctx = Context()
image_account_id = get_account_id_from_image_uri(source_image_uri)
image_region = get_region_from_image_uri(source_image_uri)
login_to_ecr_registry(ctx, image_account_id, image_region)
ctx.run(f"docker pull {source_image_uri}")
image_repo_uri, image_tag = source_image_uri.split(":")
_, image_repo_name = image_repo_uri.split("/")
test_image_repo_name = f"beta-{image_repo_name}"
if not ecr_utils.ecr_repo_exists(test_ecr_client, test_image_repo_name):
raise ecr_utils.ECRRepoDoesNotExist(
f"Repo named {test_image_repo_name} does not exist in {test_region} on the account {test_account_id}"
)
test_image_uri = (source_image_uri.replace(
image_region,
test_region).replace(image_repo_name, test_image_repo_name).replace(
image_account_id, test_account_id))
ctx.run(f"docker tag {source_image_uri} {test_image_uri}")
login_to_ecr_registry(ctx, test_account_id, test_region)
ctx.run(f"docker push {test_image_uri}")
return test_image_uri
def test_ecr_scan(image, ecr_client, sts_client, region):
"""
Run ECR Scan Tool on an image being tested, and raise Error if vulnerabilities found
1. Start Scan.
2. For 5 minutes (Run DescribeImages):
(We run this for 5 minutes because the Scan is expected to complete in about 2 minutes, though no
analysis has been performed on exactly how long the Scan takes for a DLC image. Therefore we also
have a 3 minute buffer beyond the expected amount of time taken.)
3.1. If imageScanStatus == COMPLETE: exit loop
3.2. If imageScanStatus == IN_PROGRESS or AttributeNotFound(imageScanStatus): continue loop
3.3. If imageScanStatus == FAILED: raise RuntimeError
4. If DescribeImages.imageScanStatus != COMPLETE: raise TimeOutError
5. assert imageScanFindingsSummary.findingSeverityCounts.HIGH/CRITICAL == 0
:param image: str Image URI for image to be tested
:param ecr_client: boto3 Client for ECR
:param sts_client: boto3 Client for STS
:param region: str Name of region where test is executed
"""
test_account_id = sts_client.get_caller_identity().get("Account")
image_account_id = get_account_id_from_image_uri(image)
if image_account_id != test_account_id:
image_repo_uri, image_tag = image.split(":")
_, image_repo_name = image_repo_uri.split("/")
target_image_repo_name = f"beta-{image_repo_name}"
image = ecr_utils.reupload_image_to_test_ecr(image,
target_image_repo_name,
region)
minimum_sev_threshold = get_minimum_sev_threshold_level(image)
LOGGER.info(f"Severity threshold level is {minimum_sev_threshold}")
run_scan(ecr_client, image)
scan_results = ecr_utils.get_ecr_image_scan_results(
ecr_client, image, minimum_vulnerability=minimum_sev_threshold)
scan_results = ecr_utils.populate_ecr_scan_with_web_scraper_results(
image, scan_results)
ecr_image_vulnerability_list = ScanVulnerabilityList(
minimum_severity=CVESeverity[minimum_sev_threshold])
ecr_image_vulnerability_list.construct_allowlist_from_ecr_scan_result(
scan_results)
remaining_vulnerabilities = ecr_image_vulnerability_list
# TODO: Once this feature is enabled, remove "if" condition and second assertion statement
# TODO: Ensure this works on the canary tags before removing feature flag
if is_image_covered_by_allowlist_feature(image):
upgraded_image_vulnerability_list, image_scan_allowlist = fetch_other_vulnerability_lists(
image, ecr_client, minimum_sev_threshold)
s3_bucket_name = ECR_SCAN_HELPER_BUCKET
## In case new vulnerabilities are found conduct failure routine
newly_found_vulnerabilities = ecr_image_vulnerability_list - image_scan_allowlist
if newly_found_vulnerabilities:
failure_routine_summary = conduct_failure_routine(
image,
image_scan_allowlist,
ecr_image_vulnerability_list,
upgraded_image_vulnerability_list,
s3_bucket_name,
)
(
s3_filename_for_fixable_list,
s3_filename_for_non_fixable_list,
) = process_failure_routine_summary_and_store_data_in_s3(
failure_routine_summary, s3_bucket_name)
assert not newly_found_vulnerabilities, (
f"""Found {len(failure_routine_summary["fixable_vulnerabilities"])} fixable vulnerabilites """
f"""and {len(failure_routine_summary["non_fixable_vulnerabilities"])} non fixable vulnerabilites. """
f"""Refer to files s3://{s3_bucket_name}/{s3_filename_for_fixable_list}, s3://{s3_bucket_name}/{s3_filename_for_non_fixable_list}, """
f"""s3://{s3_bucket_name}/{failure_routine_summary["s3_filename_for_current_image_ecr_scan_list"]} and s3://{s3_bucket_name}/{failure_routine_summary["s3_filename_for_allowlist"]}."""
)
## In case there is no new vulnerability but the allowlist is outdated conduct failure routine
vulnerabilities_that_can_be_fixed = image_scan_allowlist - upgraded_image_vulnerability_list
if vulnerabilities_that_can_be_fixed:
failure_routine_summary = conduct_failure_routine(
image,
image_scan_allowlist,
ecr_image_vulnerability_list,
upgraded_image_vulnerability_list,
s3_bucket_name,
)
(
s3_filename_for_fixable_list,
s3_filename_for_non_fixable_list,
) = process_failure_routine_summary_and_store_data_in_s3(
failure_routine_summary, s3_bucket_name)
assert not vulnerabilities_that_can_be_fixed, (
f"""Allowlist is Outdated!! Found {len(failure_routine_summary["fixable_vulnerabilities"])} fixable vulnerabilites """
f"""and {len(failure_routine_summary["non_fixable_vulnerabilities"])} non fixable vulnerabilites. """
f"""Refer to files s3://{s3_bucket_name}/{s3_filename_for_fixable_list}, s3://{s3_bucket_name}/{s3_filename_for_non_fixable_list}, """
f"""s3://{s3_bucket_name}/{failure_routine_summary["s3_filename_for_current_image_ecr_scan_list"]} and s3://{s3_bucket_name}/{failure_routine_summary["s3_filename_for_allowlist"]}."""
)
return
common_ecr_scan_allowlist = ScanVulnerabilityList(
minimum_severity=CVESeverity[minimum_sev_threshold])
common_ecr_scan_allowlist_path = os.path.join(
os.sep, get_repository_local_path(), "data",
"common-ecr-scan-allowlist.json")
if os.path.exists(common_ecr_scan_allowlist_path):
common_ecr_scan_allowlist.construct_allowlist_from_file(
common_ecr_scan_allowlist_path)
remaining_vulnerabilities = remaining_vulnerabilities - common_ecr_scan_allowlist
if remaining_vulnerabilities:
assert not remaining_vulnerabilities.vulnerability_list, (
f"The following vulnerabilities need to be fixed on {image}:\n"
f"{json.dumps(remaining_vulnerabilities.vulnerability_list, indent=4)}"
)
def test_ecr_scan(image, ecr_client, sts_client, region):
"""
Run ECR Scan Tool on an image being tested, and raise Error if vulnerabilities found
1. Start Scan.
2. For 5 minutes (Run DescribeImages):
(We run this for 5 minutes because the Scan is expected to complete in about 2 minutes, though no
analysis has been performed on exactly how long the Scan takes for a DLC image. Therefore we also
have a 3 minute buffer beyond the expected amount of time taken.)
3.1. If imageScanStatus == COMPLETE: exit loop
3.2. If imageScanStatus == IN_PROGRESS or AttributeNotFound(imageScanStatus): continue loop
3.3. If imageScanStatus == FAILED: raise RuntimeError
4. If DescribeImages.imageScanStatus != COMPLETE: raise TimeOutError
5. assert imageScanFindingsSummary.findingSeverityCounts.HIGH/CRITICAL == 0
:param image: str Image URI for image to be tested
:param ecr_client: boto3 Client for ECR
:param sts_client: boto3 Client for STS
:param region: str Name of region where test is executed
"""
test_account_id = sts_client.get_caller_identity().get("Account")
image_account_id = get_account_id_from_image_uri(image)
image_region = get_region_from_image_uri(image)
image_repo_name, original_image_tag = get_repository_and_tag_from_image_uri(image)
additional_image_tags = get_all_the_tags_of_an_image_from_ecr(ecr_client, image)
if not is_image_available_locally(image):
LOGGER.info(f"Image {image} not available locally!! Pulling the image...")
login_to_ecr_registry(Context(), image_account_id, image_region)
run(f"docker pull {image}")
if not is_image_available_locally(image):
raise RuntimeError("Image shown as not available even after pulling")
for additional_tag in additional_image_tags:
image_uri_with_new_tag = image.replace(original_image_tag, additional_tag)
run(f"docker tag {image} {image_uri_with_new_tag}", hide=True)
if image_account_id != test_account_id:
original_image = image
target_image_repo_name = f"beta-{image_repo_name}"
for additional_tag in additional_image_tags:
image_uri_with_new_tag = original_image.replace(original_image_tag, additional_tag)
new_image_uri = ecr_utils.reupload_image_to_test_ecr(image_uri_with_new_tag, target_image_repo_name, region)
if image_uri_with_new_tag == original_image:
image = new_image_uri
minimum_sev_threshold = get_minimum_sev_threshold_level(image)
LOGGER.info(f"Severity threshold level is {minimum_sev_threshold}")
run_scan(ecr_client, image)
scan_results = ecr_utils.get_ecr_image_scan_results(ecr_client, image, minimum_vulnerability=minimum_sev_threshold)
scan_results = ecr_utils.populate_ecr_scan_with_web_scraper_results(image, scan_results)
ecr_image_vulnerability_list = ScanVulnerabilityList(minimum_severity=CVESeverity[minimum_sev_threshold])
ecr_image_vulnerability_list.construct_allowlist_from_ecr_scan_result(scan_results)
remaining_vulnerabilities = ecr_image_vulnerability_list
if not is_image_covered_by_allowlist_feature(image):
if is_canary_context():
pytest.skip("Skipping the test on the canary.")
common_ecr_scan_allowlist = ScanVulnerabilityList(minimum_severity=CVESeverity[minimum_sev_threshold])
common_ecr_scan_allowlist_path = os.path.join(
os.sep, get_repository_local_path(), "data", "common-ecr-scan-allowlist.json"
)
if os.path.exists(common_ecr_scan_allowlist_path):
common_ecr_scan_allowlist.construct_allowlist_from_file(common_ecr_scan_allowlist_path)
remaining_vulnerabilities = remaining_vulnerabilities - common_ecr_scan_allowlist
if remaining_vulnerabilities:
assert not remaining_vulnerabilities.vulnerability_list, (
f"The following vulnerabilities need to be fixed on {image}:\n"
f"{json.dumps(remaining_vulnerabilities.vulnerability_list, indent=4)}"
)
return
upgraded_image_vulnerability_list, image_scan_allowlist = fetch_other_vulnerability_lists(
image, ecr_client, minimum_sev_threshold
)
s3_bucket_name = ECR_SCAN_HELPER_BUCKET
## In case new vulnerabilities (fixable or non-fixable) are found, then conduct failure routine
newly_found_vulnerabilities = ecr_image_vulnerability_list - image_scan_allowlist
# In case there is no new vulnerability but the allowlist is outdated
vulnerabilities_that_can_be_fixed = image_scan_allowlist - upgraded_image_vulnerability_list
if newly_found_vulnerabilities or vulnerabilities_that_can_be_fixed:
failure_routine_summary = conduct_failure_routine(
image,
image_scan_allowlist,
ecr_image_vulnerability_list,
upgraded_image_vulnerability_list,
s3_bucket_name,
)
(
s3_filename_for_fixable_list,
s3_filename_for_non_fixable_list,
) = process_failure_routine_summary_and_store_data_in_s3(failure_routine_summary, s3_bucket_name)
prepend_message = "Found new vulnerabilities in image." if newly_found_vulnerabilities else "Allowlist is outdated."
display_message = prepend_message + " " + (
f"""Found {len(failure_routine_summary["fixable_vulnerabilities"])} fixable vulnerabilites """
f"""and {len(failure_routine_summary["non_fixable_vulnerabilities"])} non fixable vulnerabilites. """
f"""Refer to files s3://{s3_bucket_name}/{s3_filename_for_fixable_list}, s3://{s3_bucket_name}/{s3_filename_for_non_fixable_list}, """
f"""s3://{s3_bucket_name}/{failure_routine_summary["s3_filename_for_current_image_ecr_scan_list"]} and s3://{s3_bucket_name}/{failure_routine_summary["s3_filename_for_allowlist"]}."""
)
if is_canary_context():
LOGGER.error(display_message)
pytest.skip("Skipping the test failure on the canary.")
else:
raise RuntimeError(display_message)
