def test_generate_coverage_doc(): """ Test generating the test coverage doc """ test_coverage_file = get_test_coverage_file_path() ctx = Context() # Set DLC_IMAGES to 'test' to avoid image names affecting function metadata (due to parametrization) # Set CODEBUILD_RESOLVED_SOURCE_VERSION to test for ease of running this test locally ctx.run( "export DLC_IMAGES='' && export CODEBUILD_RESOLVED_SOURCE_VERSION='test' && export BUILD_CONTEXT=''" "&& pytest -s --collect-only --generate-coverage-doc --ignore=container_tests/", hide=True, ) # Ensure that the coverage report is created assert os.path.exists(test_coverage_file), f"Cannot find test coverage report file {test_coverage_file}" # Write test coverage file to S3 if is_mainline_context(): client = boto3.client("s3") with open(test_coverage_file, "rb") as test_file: try: client.put_object(Bucket=TEST_COVERAGE_REPORT_BUCKET, Key=os.path.basename(test_coverage_file), Body=test_file) except ClientError as e: LOGGER.error(f"Unable to upload report to bucket {TEST_COVERAGE_REPORT_BUCKET}. Error: {e}") raise
def run_smdebug_test( image_uri, ec2_connection, region, docker_executable="docker", container_name="smdebug", test_script=SMDEBUG_SCRIPT, logfile="output.log", ): framework = get_framework_from_image_uri(image_uri) container_test_local_dir = os.path.join("$HOME", "container_tests") ec2_connection.run(f"$(aws ecr get-login --no-include-email --region {region})", hide=True) ec2_connection.run( f"{docker_executable} run --name {container_name} -v " f"{container_test_local_dir}:{os.path.join(os.sep, 'test')} -itd {image_uri}", hide=True, ) try: test_output = ec2_connection.run( f"{docker_executable} exec --user root {container_name} " f"/bin/bash -c '{test_script} {framework}' | tee {logfile}", hide=True, warn=True, timeout=3000, ) except Exception: debug_output = ec2_connection.run(f"cat {logfile}") LOGGER.error(f"Caught exception while trying to run test via fabric. Output: {debug_output.stdout}") raise # LOGGER.info(test_output.stdout) # Uncomment this line for a complete log dump assert test_output.ok, f"SMDebug tests failed. Output:\n{test_output.stdout}"
def test_oss_compliance(image): """ Run oss compliance check on a container to check if license attribution files exist. And upload source of third party packages to S3 bucket. """ THIRD_PARTY_SOURCE_CODE_BUCKET = "aws-dlinfra-licenses" THIRD_PARTY_SOURCE_CODE_BUCKET_PATH = "third_party_source_code" file = "THIRD_PARTY_SOURCE_CODE_URLS" container_name = get_container_name("oss_compliance", image) context = Context() local_repo_path = get_repository_local_path() start_container(container_name, image, context) # run compliance test to make sure license attribution files exists. testOSSCompliance is copied as part of Dockerfile run_cmd_on_container(container_name, context, "/usr/local/bin/testOSSCompliance /root") try: context.run( f"docker cp {container_name}:/root/{file} {os.path.join(local_repo_path, file)}" ) finally: context.run(f"docker rm -f {container_name}", hide=True) s3_resource = boto3.resource("s3") with open(os.path.join(local_repo_path, file)) as source_code_file: for line in source_code_file: name, version, url = line.split(" ") file_name = f"{name}_v{version}_source_code" s3_object_path = f"{THIRD_PARTY_SOURCE_CODE_BUCKET_PATH}/{file_name}.tar.gz" local_file_path = os.path.join(local_repo_path, file_name) for i in range(3): try: if not os.path.isdir(local_file_path): context.run( f"git clone {url.rstrip()} {local_file_path}") context.run( f"tar -czvf {local_file_path}.tar.gz {local_file_path}" ) except Exception as e: time.sleep(1) if i == 2: LOGGER.error(f"Unable to clone git repo. Error: {e}") raise continue try: if os.path.exists(f"{local_file_path}.tar.gz"): LOGGER.info(f"Uploading package to s3 bucket: {line}") s3_resource.Object(THIRD_PARTY_SOURCE_CODE_BUCKET, s3_object_path).load() except botocore.exceptions.ClientError as e: if e.response["Error"]["Code"] == "404": try: # using aws cli as using boto3 expects to upload folder by iterating through each file instead of entire folder. context.run( f"aws s3 cp {local_file_path}.tar.gz s3://{THIRD_PARTY_SOURCE_CODE_BUCKET}/{s3_object_path}" ) object = s3_resource.Bucket( THIRD_PARTY_SOURCE_CODE_BUCKET).Object( s3_object_path) object.Acl().put(ACL="public-read") except ClientError as e: LOGGER.error( f"Unable to upload source code to bucket {THIRD_PARTY_SOURCE_CODE_BUCKET}. Error: {e}" ) raise else: LOGGER.error( f"Unable to check if source code is present on bucket {THIRD_PARTY_SOURCE_CODE_BUCKET}. Error: {e}" ) raise
def test_ecr_scan(image, ecr_client, sts_client, region): """ Run ECR Scan Tool on an image being tested, and raise Error if vulnerabilities found 1. Start Scan. 2. For 5 minutes (Run DescribeImages): (We run this for 5 minutes because the Scan is expected to complete in about 2 minutes, though no analysis has been performed on exactly how long the Scan takes for a DLC image. Therefore we also have a 3 minute buffer beyond the expected amount of time taken.) 3.1. If imageScanStatus == COMPLETE: exit loop 3.2. If imageScanStatus == IN_PROGRESS or AttributeNotFound(imageScanStatus): continue loop 3.3. If imageScanStatus == FAILED: raise RuntimeError 4. If DescribeImages.imageScanStatus != COMPLETE: raise TimeOutError 5. assert imageScanFindingsSummary.findingSeverityCounts.HIGH/CRITICAL == 0 :param image: str Image URI for image to be tested :param ecr_client: boto3 Client for ECR :param sts_client: boto3 Client for STS :param region: str Name of region where test is executed """ test_account_id = sts_client.get_caller_identity().get("Account") image_account_id = get_account_id_from_image_uri(image) image_region = get_region_from_image_uri(image) image_repo_name, original_image_tag = get_repository_and_tag_from_image_uri(image) additional_image_tags = get_all_the_tags_of_an_image_from_ecr(ecr_client, image) if not is_image_available_locally(image): LOGGER.info(f"Image {image} not available locally!! Pulling the image...") login_to_ecr_registry(Context(), image_account_id, image_region) run(f"docker pull {image}") if not is_image_available_locally(image): raise RuntimeError("Image shown as not available even after pulling") for additional_tag in additional_image_tags: image_uri_with_new_tag = image.replace(original_image_tag, additional_tag) run(f"docker tag {image} {image_uri_with_new_tag}", hide=True) if image_account_id != test_account_id: original_image = image target_image_repo_name = f"beta-{image_repo_name}" for additional_tag in additional_image_tags: image_uri_with_new_tag = original_image.replace(original_image_tag, additional_tag) new_image_uri = ecr_utils.reupload_image_to_test_ecr(image_uri_with_new_tag, target_image_repo_name, region) if image_uri_with_new_tag == original_image: image = new_image_uri minimum_sev_threshold = get_minimum_sev_threshold_level(image) LOGGER.info(f"Severity threshold level is {minimum_sev_threshold}") run_scan(ecr_client, image) scan_results = ecr_utils.get_ecr_image_scan_results(ecr_client, image, minimum_vulnerability=minimum_sev_threshold) scan_results = ecr_utils.populate_ecr_scan_with_web_scraper_results(image, scan_results) ecr_image_vulnerability_list = ScanVulnerabilityList(minimum_severity=CVESeverity[minimum_sev_threshold]) ecr_image_vulnerability_list.construct_allowlist_from_ecr_scan_result(scan_results) remaining_vulnerabilities = ecr_image_vulnerability_list if not is_image_covered_by_allowlist_feature(image): if is_canary_context(): pytest.skip("Skipping the test on the canary.") common_ecr_scan_allowlist = ScanVulnerabilityList(minimum_severity=CVESeverity[minimum_sev_threshold]) common_ecr_scan_allowlist_path = os.path.join( os.sep, get_repository_local_path(), "data", "common-ecr-scan-allowlist.json" ) if os.path.exists(common_ecr_scan_allowlist_path): common_ecr_scan_allowlist.construct_allowlist_from_file(common_ecr_scan_allowlist_path) remaining_vulnerabilities = remaining_vulnerabilities - common_ecr_scan_allowlist if remaining_vulnerabilities: assert not remaining_vulnerabilities.vulnerability_list, ( f"The following vulnerabilities need to be fixed on {image}:\n" f"{json.dumps(remaining_vulnerabilities.vulnerability_list, indent=4)}" ) return upgraded_image_vulnerability_list, image_scan_allowlist = fetch_other_vulnerability_lists( image, ecr_client, minimum_sev_threshold ) s3_bucket_name = ECR_SCAN_HELPER_BUCKET ## In case new vulnerabilities (fixable or non-fixable) are found, then conduct failure routine newly_found_vulnerabilities = ecr_image_vulnerability_list - image_scan_allowlist # In case there is no new vulnerability but the allowlist is outdated vulnerabilities_that_can_be_fixed = image_scan_allowlist - upgraded_image_vulnerability_list if newly_found_vulnerabilities or vulnerabilities_that_can_be_fixed: failure_routine_summary = conduct_failure_routine( image, image_scan_allowlist, ecr_image_vulnerability_list, upgraded_image_vulnerability_list, s3_bucket_name, ) ( s3_filename_for_fixable_list, s3_filename_for_non_fixable_list, ) = process_failure_routine_summary_and_store_data_in_s3(failure_routine_summary, s3_bucket_name) prepend_message = "Found new vulnerabilities in image." if newly_found_vulnerabilities else "Allowlist is outdated." display_message = prepend_message + " " + ( f"""Found {len(failure_routine_summary["fixable_vulnerabilities"])} fixable vulnerabilites """ f"""and {len(failure_routine_summary["non_fixable_vulnerabilities"])} non fixable vulnerabilites. """ f"""Refer to files s3://{s3_bucket_name}/{s3_filename_for_fixable_list}, s3://{s3_bucket_name}/{s3_filename_for_non_fixable_list}, """ f"""s3://{s3_bucket_name}/{failure_routine_summary["s3_filename_for_current_image_ecr_scan_list"]} and s3://{s3_bucket_name}/{failure_routine_summary["s3_filename_for_allowlist"]}.""" ) if is_canary_context(): LOGGER.error(display_message) pytest.skip("Skipping the test failure on the canary.") else: raise RuntimeError(display_message)