def get_ec2_instance_type(default, processor, filter_function=lambda x: x): """ Get EC2 instance type from associated EC2_[CPU|GPU]_INSTANCE_TYPE env variable, or set it to a default for contexts where the variable is not present (i.e. PR, Nightly, local testing) :param default: Default instance type to use - Should never be p3dn :param processor: "cpu" or "gpu" :param filter_function: filter_function(instance_type_list) A function that takes the list to be generated by the logic of the get_ec2_instance_type function, and filters the list to only produce "acceptable" instances. For example, this can be a function that only returns multi-gpu instance types from a given list of instance types. :return: one item list of instance type -- this is used to parametrize tests, and parameter is required to be a list. """ allowed_processors = ("cpu", "gpu", "neuron") if processor not in allowed_processors: raise RuntimeError( f"Aborting EC2 test run. Unrecognized processor type {processor}. " f"Please choose from {allowed_processors}") if default in HEAVY_INSTANCE_LIST: raise RuntimeError( f"Default instance type should never be one of {HEAVY_INSTANCE_LIST}, but it is {default}" ) instance_type = os.getenv(f"EC2_{processor.upper()}_INSTANCE_TYPE") if not instance_type and is_mainline_context(): return [] instance_list = filter_function([instance_type] if instance_type else []) if not instance_list: instance_list = [default] return instance_list
def test_generate_coverage_doc(): """ Test generating the test coverage doc """ test_coverage_file = get_test_coverage_file_path() ctx = Context() # Set DLC_IMAGES to 'test' to avoid image names affecting function metadata (due to parametrization) # Set CODEBUILD_RESOLVED_SOURCE_VERSION to test for ease of running this test locally ctx.run( "export DLC_IMAGES='' && export CODEBUILD_RESOLVED_SOURCE_VERSION='test' && export BUILD_CONTEXT=''" "&& pytest -s --collect-only --generate-coverage-doc --ignore=container_tests/", hide=True, ) # Ensure that the coverage report is created assert os.path.exists(test_coverage_file), f"Cannot find test coverage report file {test_coverage_file}" # Write test coverage file to S3 if is_mainline_context(): client = boto3.client("s3") with open(test_coverage_file, "rb") as test_file: try: client.put_object(Bucket=TEST_COVERAGE_REPORT_BUCKET, Key=os.path.basename(test_coverage_file), Body=test_file) except ClientError as e: LOGGER.error(f"Unable to upload report to bucket {TEST_COVERAGE_REPORT_BUCKET}. Error: {e}") raise
def get_ec2_instance_type(default, processor, disable_large_type=True): """ Get EC2 instance type from associated EC2_[CPU|GPU]_INSTANCE_TYPE env variable, or set it to a default for contexts where the variable is not present (i.e. PR, Nightly, local testing) :param default: Default instance type to use - Should never be p3dn :param processor: "cpu" or "gpu" :param disable_large_type: Boolean to determine whether or not to run tests on p3dn. If set to true, default gpu instance type will be used. :return: one item list of instance type -- this is used to parametrize tests, and parameter is required to be a list. """ allowed_processors = ("cpu", "gpu") p4d = "p4d.24xlarge" p3dn = "p3dn.24xlarge" if processor not in allowed_processors: raise RuntimeError( f"Aborting EC2 test run. Unrecognized processor type {processor}. " f"Please choose from {allowed_processors}") if default in [p3dn, p4d]: raise RuntimeError( "Default instance type should never be p3dn.24xlarge or p4d.24xlarge" ) instance_type = os.getenv(f"EC2_{processor.upper()}_INSTANCE_TYPE") if not instance_type and is_mainline_context(): return [] instance_type = default if disable_large_type and instance_type in [p3dn, p4d]: instance_type = default return [instance_type]
def get_ec2_accelerator_type(default, processor): """ Get EC2 instance type from associated EC2_EIA_INSTANCE_TYPE env variable, or set it to a default for contexts where the variable is not present (i.e. PR, Nightly, local testing) :param default: Default accelerator instance type to use :param processor: "eia" :return: one item list of instance type -- this is used to parametrize tests, and parameter is required to be a list. """ allowed_processors = ("eia", ) if processor not in allowed_processors: raise RuntimeError( f"Aborting EC2 test run. Unrecognized processor type {processor}. " f"Please choose from {allowed_processors}") accelerator_type = os.getenv(f"EC2_{processor.upper()}_INSTANCE_TYPE") if not accelerator_type: if is_mainline_context(): return [] return [default] return [accelerator_type]
from test.test_utils import ( is_mainline_context, is_rc_test_context, get_framework_and_version_from_tag, get_container_name, get_processor_from_image_uri, is_tf_version, LOGGER, ) @pytest.mark.usefixtures("sagemaker_only", "huggingface") @pytest.mark.integration("smprofiler") @pytest.mark.model("N/A") @pytest.mark.skipif(not is_mainline_context() and not is_rc_test_context(), reason="Mainline only test") def test_sm_profiler_pt(pytorch_training): processor = get_processor_from_image_uri(pytorch_training) if processor not in ("cpu", "gpu"): pytest.skip(f"Processor {processor} not supported. Skipping test.") ctx = Context() profiler_tests_dir = os.path.join( os.getenv("CODEBUILD_SRC_DIR"), get_container_name("smprof", pytorch_training), "smprofiler_tests") ctx.run(f"mkdir -p {profiler_tests_dir}", hide=True) # Download sagemaker-tests zip sm_tests_zip = "sagemaker-tests.zip"
""" Get the latest package version available on pypi for a package. It is retried multiple times in case there are transient failures in executing the command. :param package: str Name of the package whose latest version must be retrieved :return: tuple(command_success: bool, latest_version_value: str) """ pypi_package_info = requests.get(f"https://pypi.org/pypi/{package}/json") data = json.loads(pypi_package_info.text) versions = data["releases"].keys() return str(max(Version(v) for v in versions)) @pytest.mark.model("N/A") @pytest.mark.skipif(not is_dlc_cicd_context(), reason="Skipping test because it is not running in dlc cicd infra") @pytest.mark.skipif(not is_mainline_context(), reason="Skipping the test to decrease the number of calls to the Safety Check DB. " "Test will be executed in the 'mainline' pipeline only") def test_safety(image): """ Runs safety check on a container with the capability to ignore safety issues that cannot be fixed, and only raise error if an issue is fixable. """ from dlc.safety_check import SafetyCheck safety_check = SafetyCheck() repo_name, image_tag = image.split('/')[-1].split(':') ignore_ids_list = _get_safety_ignore_list(image) sep = " -i " ignore_str = "" if not ignore_ids_list else f"{sep}{sep.join(ignore_ids_list)}"
:return: tuple(command_success: bool, latest_version_value: str) """ pypi_package_info = requests.get(f"https://pypi.org/pypi/{package}/json") data = json.loads(pypi_package_info.text) versions = data["releases"].keys() return str(max(Version(v) for v in versions)) @pytest.mark.usefixtures("sagemaker") @pytest.mark.model("N/A") @pytest.mark.canary("Run safety tests regularly on production images") @pytest.mark.skipif( not is_dlc_cicd_context(), reason="Skipping test because it is not running in dlc cicd infra") @pytest.mark.skipif( not (is_mainline_context() or is_safety_test_context() or (is_canary_context() and is_time_for_canary_safety_scan())), reason= ("Skipping the test to decrease the number of calls to the Safety Check DB. " "Test will be executed in the 'mainline' pipeline and canaries pipeline." ), ) def test_safety(image): """ Runs safety check on a container with the capability to ignore safety issues that cannot be fixed, and only raise error if an issue is fixable. """ from dlc.safety_check import SafetyCheck safety_check = SafetyCheck()
expected_versions = [v + 1 for v in expected_versions] expected_versions[0] = 1 assert 2 not in actual_versions, ( f"DLC v2.0 is deprecated in PyTorch 1.6.0 gpu containers, but found major version 2 " f"in one of the Dockerfiles. Please inspect {versions}") # Note: If, for example, we find 3 dockerfiles with the same framework major/minor version, same processor, # and same python major/minor version, we will expect DLC major versions 1, 2, and 3. If an exception needs to be # made to this rule, please see the above handling of TF2.3 as an example. assert actual_versions == expected_versions, ( f"Found DLC major versions {actual_versions} but expected {expected_versions} for " f"{framework} {job_type} {processor}. Full version info: {versions}. Py version: {python_major_minor_version}" ) @pytest.mark.skipif(not test_utils.is_mainline_context(), reason="This test only applies to Release Candidate images" ) @pytest.mark.usefixtures("sagemaker") @pytest.mark.integration("dlc_nightly_feature_label") @pytest.mark.model("N/A") def test_dlc_nightly_feature_labels(image, region): """ Test to ensure that nightly feature labels are not applied on prod DLCs :param image: :return: """ image_labels = test_utils.get_labels_from_ecr_image(image, region) nightly_labels_on_image = [ label.value for label in test_utils.NightlyFeatureLabel.__members__.values()
:param package: str Name of the package whose latest version must be retrieved :return: tuple(command_success: bool, latest_version_value: str) """ pypi_package_info = requests.get(f"https://pypi.org/pypi/{package}/json") data = json.loads(pypi_package_info.text) versions = data["releases"].keys() return str(max(Version(v) for v in versions)) @pytest.mark.model("N/A") @pytest.mark.skipif( not is_dlc_cicd_context(), reason="Skipping test because it is not running in dlc cicd infra") @pytest.mark.skipif( not is_mainline_context(), reason= "Skipping the test to decrease the number of calls to the Safety Check DB. " "Test will be executed in the 'mainline' pipeline only") def test_safety(image): """ Runs safety check on a container with the capability to ignore safety issues that cannot be fixed, and only raise error if an issue is fixable. """ from dlc.safety_check import SafetyCheck safety_check = SafetyCheck() repo_name, image_tag = image.split('/')[-1].split(':') ignore_ids_list = _get_safety_ignore_list(image) sep = " -i " ignore_str = "" if not ignore_ids_list else f"{sep}{sep.join(ignore_ids_list)}"
from invoke.context import Context from invoke.exceptions import UnexpectedExit from test.test_utils import ( is_mainline_context, get_framework_and_version_from_tag, get_container_name, get_processor_from_image_uri, is_tf_version, LOGGER, ) @pytest.mark.integration("smprofiler") @pytest.mark.model("N/A") @pytest.mark.skipif(not is_mainline_context(), reason="Mainline only test") def test_sm_profiler_pt(pytorch_training): processor = get_processor_from_image_uri(pytorch_training) if processor not in ("cpu", "gpu"): pytest.skip(f"Processor {processor} not supported. Skipping test.") ctx = Context() profiler_tests_dir = os.path.join( os.getenv("CODEBUILD_SRC_DIR"), get_container_name("smprof", pytorch_training), "smprofiler_tests") ctx.run(f"mkdir -p {profiler_tests_dir}", hide=True) # Download sagemaker-tests zip sm_tests_zip = "sagemaker-tests.zip" ctx.run(
# TODO: Remove this once we have whitelisted appropriate LOW/MEDIUM vulnerabilities if not (vulnerability_severity.get("CRITICAL") or vulnerability_severity.get("HIGH")): return raise DependencyCheckFailure( f"Unrecognized CVEs have been reported : {vulnerability_severity}. " f"Allowed vulnerabilities are {allowed_vulnerabilities or None}. Please see " f"{dependency_check_report} for more details.") @pytest.mark.model("N/A") @pytest.mark.canary("Run dependency tests regularly on production images") @pytest.mark.parametrize("ec2_instance_type", ["c5.4xlarge"], indirect=True) @pytest.mark.skipif( not (is_nightly_context() or is_mainline_context() or (is_canary_context() and is_time_for_canary_safety_scan())), reason="Do not run dependency check on PR tests. " "Executing test in canaries pipeline during only a limited period of time." ) def test_dependency_check_cpu(cpu, ec2_connection): _run_dependency_check_test(cpu, ec2_connection, "cpu") @pytest.mark.model("N/A") @pytest.mark.canary("Run dependency tests regularly on production images") @pytest.mark.parametrize("ec2_instance_type", ["p3.2xlarge"], indirect=True) @pytest.mark.skipif( not (is_nightly_context() or is_mainline_context() or (is_canary_context() and is_time_for_canary_safety_scan())), reason="Do not run dependency check on PR tests. "
:param package: str Name of the package whose latest version must be retrieved :return: tuple(command_success: bool, latest_version_value: str) """ pypi_package_info = requests.get(f"https://pypi.org/pypi/{package}/json") data = json.loads(pypi_package_info.text) versions = data["releases"].keys() return str(max(Version(v) for v in versions)) @pytest.mark.model("N/A") @pytest.mark.canary("Run safety tests regularly on production images") @pytest.mark.skipif( not is_dlc_cicd_context(), reason="Skipping test because it is not running in dlc cicd infra") @pytest.mark.skipif( not (is_mainline_context() or (is_canary_context() and is_time_for_canary_safety_scan())), reason= ("Skipping the test to decrease the number of calls to the Safety Check DB. " "Test will be executed in the 'mainline' pipeline and canaries pipeline.") ) def test_safety(image): """ Runs safety check on a container with the capability to ignore safety issues that cannot be fixed, and only raise error if an issue is fixable. """ from dlc.safety_check import SafetyCheck safety_check = SafetyCheck() repo_name, image_tag = image.split('/')[-1].split(':') ignore_ids_list = _get_safety_ignore_list(image)
from botocore.exceptions import ClientError from invoke.context import Context from test.test_utils import LOGGER, is_mainline_context, is_graviton_architecture from test.test_utils.test_reporting import get_test_coverage_file_path ACCOUNT_ID = os.getenv("ACCOUNT_ID", boto3.client("sts").get_caller_identity().get("Account")) TEST_COVERAGE_REPORT_BUCKET = f"dlc-test-coverage-reports-{ACCOUNT_ID}" @pytest.mark.quick_checks @pytest.mark.integration("Generating this coverage doc") @pytest.mark.model("N/A") @pytest.mark.skipif( (is_mainline_context() and is_graviton_architecture()), reason="Skipping the test for Graviton image build in mainline context as ARM image is used as a base", ) def test_generate_coverage_doc(): """ Test generating the test coverage doc """ test_coverage_file = get_test_coverage_file_path() ctx = Context() # Set DLC_IMAGES to 'test' to avoid image names affecting function metadata (due to parametrization) # Set CODEBUILD_RESOLVED_SOURCE_VERSION to test for ease of running this test locally ctx.run( "export DLC_IMAGES='' && export CODEBUILD_RESOLVED_SOURCE_VERSION='test' && export BUILD_CONTEXT=''" "&& pytest -s --collect-only --generate-coverage-doc --ignore=container_tests/", hide=True,