def get_ec2_instance_type(default, processor, filter_function=lambda x: x):
"""
Get EC2 instance type from associated EC2_[CPU|GPU]_INSTANCE_TYPE env variable, or set it to a default
for contexts where the variable is not present (i.e. PR, Nightly, local testing)
:param default: Default instance type to use - Should never be p3dn
:param processor: "cpu" or "gpu"
:param filter_function: filter_function(instance_type_list) A function that takes the list to be generated by
the logic of the get_ec2_instance_type function, and filters the list to only produce "acceptable" instances.
For example, this can be a function that only returns multi-gpu instance types from a given list of instance types.
:return: one item list of instance type -- this is used to parametrize tests, and parameter is required to be
a list.
"""
allowed_processors = ("cpu", "gpu", "neuron")
if processor not in allowed_processors:
raise RuntimeError(
f"Aborting EC2 test run. Unrecognized processor type {processor}. "
f"Please choose from {allowed_processors}")
if default in HEAVY_INSTANCE_LIST:
raise RuntimeError(
f"Default instance type should never be one of {HEAVY_INSTANCE_LIST}, but it is {default}"
)
instance_type = os.getenv(f"EC2_{processor.upper()}_INSTANCE_TYPE")
if not instance_type and is_mainline_context():
return []
instance_list = filter_function([instance_type] if instance_type else [])
if not instance_list:
instance_list = [default]
return instance_list
def test_generate_coverage_doc():
"""
Test generating the test coverage doc
"""
test_coverage_file = get_test_coverage_file_path()
ctx = Context()
# Set DLC_IMAGES to 'test' to avoid image names affecting function metadata (due to parametrization)
# Set CODEBUILD_RESOLVED_SOURCE_VERSION to test for ease of running this test locally
ctx.run(
"export DLC_IMAGES='' && export CODEBUILD_RESOLVED_SOURCE_VERSION='test' && export BUILD_CONTEXT=''"
"&& pytest -s --collect-only --generate-coverage-doc --ignore=container_tests/",
hide=True,
)
# Ensure that the coverage report is created
assert os.path.exists(test_coverage_file), f"Cannot find test coverage report file {test_coverage_file}"
# Write test coverage file to S3
if is_mainline_context():
client = boto3.client("s3")
with open(test_coverage_file, "rb") as test_file:
try:
client.put_object(Bucket=TEST_COVERAGE_REPORT_BUCKET, Key=os.path.basename(test_coverage_file),
Body=test_file)
except ClientError as e:
LOGGER.error(f"Unable to upload report to bucket {TEST_COVERAGE_REPORT_BUCKET}. Error: {e}")
raise
def get_ec2_instance_type(default, processor, disable_large_type=True):
"""
Get EC2 instance type from associated EC2_[CPU|GPU]_INSTANCE_TYPE env variable, or set it to a default
for contexts where the variable is not present (i.e. PR, Nightly, local testing)
:param default: Default instance type to use - Should never be p3dn
:param processor: "cpu" or "gpu"
:param disable_large_type: Boolean to determine whether or not to run tests on p3dn. If set to true, default
gpu instance type will be used.
:return: one item list of instance type -- this is used to parametrize tests, and parameter is required to be
a list.
"""
allowed_processors = ("cpu", "gpu")
p4d = "p4d.24xlarge"
p3dn = "p3dn.24xlarge"
if processor not in allowed_processors:
raise RuntimeError(
f"Aborting EC2 test run. Unrecognized processor type {processor}. "
f"Please choose from {allowed_processors}")
if default in [p3dn, p4d]:
raise RuntimeError(
"Default instance type should never be p3dn.24xlarge or p4d.24xlarge"
)
instance_type = os.getenv(f"EC2_{processor.upper()}_INSTANCE_TYPE")
if not instance_type and is_mainline_context():
return []
instance_type = default
if disable_large_type and instance_type in [p3dn, p4d]:
instance_type = default
return [instance_type]
def get_ec2_accelerator_type(default, processor):
"""
Get EC2 instance type from associated EC2_EIA_INSTANCE_TYPE env variable, or set it to a default
for contexts where the variable is not present (i.e. PR, Nightly, local testing)
:param default: Default accelerator instance type to use
:param processor: "eia"
:return: one item list of instance type -- this is used to parametrize tests, and parameter is required to be
a list.
"""
allowed_processors = ("eia", )
if processor not in allowed_processors:
raise RuntimeError(
f"Aborting EC2 test run. Unrecognized processor type {processor}. "
f"Please choose from {allowed_processors}")
accelerator_type = os.getenv(f"EC2_{processor.upper()}_INSTANCE_TYPE")
if not accelerator_type:
if is_mainline_context():
return []
return [default]
return [accelerator_type]
from test.test_utils import (
is_mainline_context,
is_rc_test_context,
get_framework_and_version_from_tag,
get_container_name,
get_processor_from_image_uri,
is_tf_version,
LOGGER,
)
@pytest.mark.usefixtures("sagemaker_only", "huggingface")
@pytest.mark.integration("smprofiler")
@pytest.mark.model("N/A")
@pytest.mark.skipif(not is_mainline_context() and not is_rc_test_context(),
reason="Mainline only test")
def test_sm_profiler_pt(pytorch_training):
processor = get_processor_from_image_uri(pytorch_training)
if processor not in ("cpu", "gpu"):
pytest.skip(f"Processor {processor} not supported. Skipping test.")
ctx = Context()
profiler_tests_dir = os.path.join(
os.getenv("CODEBUILD_SRC_DIR"),
get_container_name("smprof", pytorch_training), "smprofiler_tests")
ctx.run(f"mkdir -p {profiler_tests_dir}", hide=True)
# Download sagemaker-tests zip
sm_tests_zip = "sagemaker-tests.zip"
"""
Get the latest package version available on pypi for a package.
It is retried multiple times in case there are transient failures in executing the command.
:param package: str Name of the package whose latest version must be retrieved
:return: tuple(command_success: bool, latest_version_value: str)
"""
pypi_package_info = requests.get(f"https://pypi.org/pypi/{package}/json")
data = json.loads(pypi_package_info.text)
versions = data["releases"].keys()
return str(max(Version(v) for v in versions))
@pytest.mark.model("N/A")
@pytest.mark.skipif(not is_dlc_cicd_context(), reason="Skipping test because it is not running in dlc cicd infra")
@pytest.mark.skipif(not is_mainline_context(),
reason="Skipping the test to decrease the number of calls to the Safety Check DB. "
"Test will be executed in the 'mainline' pipeline only")
def test_safety(image):
"""
Runs safety check on a container with the capability to ignore safety issues that cannot be fixed, and only raise
error if an issue is fixable.
"""
from dlc.safety_check import SafetyCheck
safety_check = SafetyCheck()
repo_name, image_tag = image.split('/')[-1].split(':')
ignore_ids_list = _get_safety_ignore_list(image)
sep = " -i "
ignore_str = "" if not ignore_ids_list else f"{sep}{sep.join(ignore_ids_list)}"
:return: tuple(command_success: bool, latest_version_value: str)
"""
pypi_package_info = requests.get(f"https://pypi.org/pypi/{package}/json")
data = json.loads(pypi_package_info.text)
versions = data["releases"].keys()
return str(max(Version(v) for v in versions))
@pytest.mark.usefixtures("sagemaker")
@pytest.mark.model("N/A")
@pytest.mark.canary("Run safety tests regularly on production images")
@pytest.mark.skipif(
not is_dlc_cicd_context(),
reason="Skipping test because it is not running in dlc cicd infra")
@pytest.mark.skipif(
not (is_mainline_context() or is_safety_test_context() or
(is_canary_context() and is_time_for_canary_safety_scan())),
reason=
("Skipping the test to decrease the number of calls to the Safety Check DB. "
"Test will be executed in the 'mainline' pipeline and canaries pipeline."
),
)
def test_safety(image):
"""
Runs safety check on a container with the capability to ignore safety issues that cannot be fixed, and only raise
error if an issue is fixable.
"""
from dlc.safety_check import SafetyCheck
safety_check = SafetyCheck()
expected_versions = [v + 1 for v in expected_versions]
expected_versions[0] = 1
assert 2 not in actual_versions, (
f"DLC v2.0 is deprecated in PyTorch 1.6.0 gpu containers, but found major version 2 "
f"in one of the Dockerfiles. Please inspect {versions}")
# Note: If, for example, we find 3 dockerfiles with the same framework major/minor version, same processor,
# and same python major/minor version, we will expect DLC major versions 1, 2, and 3. If an exception needs to be
# made to this rule, please see the above handling of TF2.3 as an example.
assert actual_versions == expected_versions, (
f"Found DLC major versions {actual_versions} but expected {expected_versions} for "
f"{framework} {job_type} {processor}. Full version info: {versions}. Py version: {python_major_minor_version}"
)
@pytest.mark.skipif(not test_utils.is_mainline_context(),
reason="This test only applies to Release Candidate images"
)
@pytest.mark.usefixtures("sagemaker")
@pytest.mark.integration("dlc_nightly_feature_label")
@pytest.mark.model("N/A")
def test_dlc_nightly_feature_labels(image, region):
"""
Test to ensure that nightly feature labels are not applied on prod DLCs
:param image:
:return:
"""
image_labels = test_utils.get_labels_from_ecr_image(image, region)
nightly_labels_on_image = [
label.value
for label in test_utils.NightlyFeatureLabel.__members__.values()
:param package: str Name of the package whose latest version must be retrieved
:return: tuple(command_success: bool, latest_version_value: str)
"""
pypi_package_info = requests.get(f"https://pypi.org/pypi/{package}/json")
data = json.loads(pypi_package_info.text)
versions = data["releases"].keys()
return str(max(Version(v) for v in versions))
@pytest.mark.model("N/A")
@pytest.mark.skipif(
not is_dlc_cicd_context(),
reason="Skipping test because it is not running in dlc cicd infra")
@pytest.mark.skipif(
not is_mainline_context(),
reason=
"Skipping the test to decrease the number of calls to the Safety Check DB. "
"Test will be executed in the 'mainline' pipeline only")
def test_safety(image):
"""
Runs safety check on a container with the capability to ignore safety issues that cannot be fixed, and only raise
error if an issue is fixable.
"""
from dlc.safety_check import SafetyCheck
safety_check = SafetyCheck()
repo_name, image_tag = image.split('/')[-1].split(':')
ignore_ids_list = _get_safety_ignore_list(image)
sep = " -i "
ignore_str = "" if not ignore_ids_list else f"{sep}{sep.join(ignore_ids_list)}"
from invoke.context import Context
from invoke.exceptions import UnexpectedExit
from test.test_utils import (
is_mainline_context,
get_framework_and_version_from_tag,
get_container_name,
get_processor_from_image_uri,
is_tf_version,
LOGGER,
)
@pytest.mark.integration("smprofiler")
@pytest.mark.model("N/A")
@pytest.mark.skipif(not is_mainline_context(), reason="Mainline only test")
def test_sm_profiler_pt(pytorch_training):
processor = get_processor_from_image_uri(pytorch_training)
if processor not in ("cpu", "gpu"):
pytest.skip(f"Processor {processor} not supported. Skipping test.")
ctx = Context()
profiler_tests_dir = os.path.join(
os.getenv("CODEBUILD_SRC_DIR"),
get_container_name("smprof", pytorch_training), "smprofiler_tests")
ctx.run(f"mkdir -p {profiler_tests_dir}", hide=True)
# Download sagemaker-tests zip
sm_tests_zip = "sagemaker-tests.zip"
ctx.run(
# TODO: Remove this once we have whitelisted appropriate LOW/MEDIUM vulnerabilities
if not (vulnerability_severity.get("CRITICAL")
or vulnerability_severity.get("HIGH")):
return
raise DependencyCheckFailure(
f"Unrecognized CVEs have been reported : {vulnerability_severity}. "
f"Allowed vulnerabilities are {allowed_vulnerabilities or None}. Please see "
f"{dependency_check_report} for more details.")
@pytest.mark.model("N/A")
@pytest.mark.canary("Run dependency tests regularly on production images")
@pytest.mark.parametrize("ec2_instance_type", ["c5.4xlarge"], indirect=True)
@pytest.mark.skipif(
not (is_nightly_context() or is_mainline_context() or
(is_canary_context() and is_time_for_canary_safety_scan())),
reason="Do not run dependency check on PR tests. "
"Executing test in canaries pipeline during only a limited period of time."
)
def test_dependency_check_cpu(cpu, ec2_connection):
_run_dependency_check_test(cpu, ec2_connection, "cpu")
@pytest.mark.model("N/A")
@pytest.mark.canary("Run dependency tests regularly on production images")
@pytest.mark.parametrize("ec2_instance_type", ["p3.2xlarge"], indirect=True)
@pytest.mark.skipif(
not (is_nightly_context() or is_mainline_context() or
(is_canary_context() and is_time_for_canary_safety_scan())),
reason="Do not run dependency check on PR tests. "
:param package: str Name of the package whose latest version must be retrieved
:return: tuple(command_success: bool, latest_version_value: str)
"""
pypi_package_info = requests.get(f"https://pypi.org/pypi/{package}/json")
data = json.loads(pypi_package_info.text)
versions = data["releases"].keys()
return str(max(Version(v) for v in versions))
@pytest.mark.model("N/A")
@pytest.mark.canary("Run safety tests regularly on production images")
@pytest.mark.skipif(
not is_dlc_cicd_context(),
reason="Skipping test because it is not running in dlc cicd infra")
@pytest.mark.skipif(
not (is_mainline_context() or
(is_canary_context() and is_time_for_canary_safety_scan())),
reason=
("Skipping the test to decrease the number of calls to the Safety Check DB. "
"Test will be executed in the 'mainline' pipeline and canaries pipeline.")
)
def test_safety(image):
"""
Runs safety check on a container with the capability to ignore safety issues that cannot be fixed, and only raise
error if an issue is fixable.
"""
from dlc.safety_check import SafetyCheck
safety_check = SafetyCheck()
repo_name, image_tag = image.split('/')[-1].split(':')
ignore_ids_list = _get_safety_ignore_list(image)
from botocore.exceptions import ClientError
from invoke.context import Context
from test.test_utils import LOGGER, is_mainline_context, is_graviton_architecture
from test.test_utils.test_reporting import get_test_coverage_file_path
ACCOUNT_ID = os.getenv("ACCOUNT_ID", boto3.client("sts").get_caller_identity().get("Account"))
TEST_COVERAGE_REPORT_BUCKET = f"dlc-test-coverage-reports-{ACCOUNT_ID}"
@pytest.mark.quick_checks
@pytest.mark.integration("Generating this coverage doc")
@pytest.mark.model("N/A")
@pytest.mark.skipif(
(is_mainline_context() and is_graviton_architecture()),
reason="Skipping the test for Graviton image build in mainline context as ARM image is used as a base",
)
def test_generate_coverage_doc():
"""
Test generating the test coverage doc
"""
test_coverage_file = get_test_coverage_file_path()
ctx = Context()
# Set DLC_IMAGES to 'test' to avoid image names affecting function metadata (due to parametrization)
# Set CODEBUILD_RESOLVED_SOURCE_VERSION to test for ease of running this test locally
ctx.run(
"export DLC_IMAGES='' && export CODEBUILD_RESOLVED_SOURCE_VERSION='test' && export BUILD_CONTEXT=''"
"&& pytest -s --collect-only --generate-coverage-doc --ignore=container_tests/",
hide=True,
