def test_smdebug_gpu(training, ec2_connection, region, ec2_instance_type,
gpu_only, py3_only):
if test_utils.is_image_incompatible_with_instance_type(
training, ec2_instance_type):
pytest.skip(
f"Image {training} is incompatible with instance type {ec2_instance_type}"
)
smdebug_test_timeout = 2400
if is_tf_version("1", training):
if is_nightly_context():
smdebug_test_timeout = 7200
else:
pytest.skip(
"TF1 gpu smdebug tests can take up to 2 hours, thus we are only running in nightly context"
)
run_smdebug_test(
training,
ec2_connection,
region,
ec2_instance_type,
docker_executable="nvidia-docker",
container_name="smdebug-gpu",
timeout=smdebug_test_timeout,
)
def test_smdebug_gpu(training, ec2_connection, region, ec2_instance_type, gpu_only, py3_only):
smdebug_test_timeout = 2400
if is_tf_version("1", training):
if is_nightly_context():
smdebug_test_timeout = 7200
else:
pytest.skip("TF1 gpu smdebug tests can take up to 2 hours, thus we are only running in nightly context")
run_smdebug_test(
training,
ec2_connection,
region,
ec2_instance_type,
docker_executable="nvidia-docker",
container_name="smdebug-gpu",
timeout=smdebug_test_timeout
)
port_to_forward = random.randint(49152, 65535)
if eks_utils.is_service_running(selector_name):
eks_utils.eks_forward_port_between_host_and_container(
selector_name, port_to_forward, "8500")
assert test_utils.request_tensorflow_inference(model_name=model_name,
port=port_to_forward)
except ValueError as excp:
eks_utils.LOGGER.error("Service is not running: %s", excp)
finally:
run(f"kubectl delete deployment {selector_name}")
run(f"kubectl delete service {selector_name}")
@pytest.mark.skipif(not test_utils.is_nightly_context(),
reason="Running additional model in nightly context only")
@pytest.mark.skip(
"Skipping test due to S3 permission issues. https://github.com/aws/deep-learning-containers/issues/818"
)
@pytest.mark.model("albert")
def test_eks_tensorflow_albert(tensorflow_inference):
if "eia" in tensorflow_inference or "neuron" in tensorflow_inference:
pytest.skip("Skipping EKS Test for EIA and neuron Images")
num_replicas = "1"
rand_int = random.randint(4001, 6000)
processor = "gpu" if "gpu" in tensorflow_inference else "cpu"
model_name = f"albert"
worker_instance_id,
num_gpus=num_gpus,
region=region,
)
model_name = get_tensorflow_model_name("gpu", model_name)
inference_result = request_tensorflow_inference(
model_name, ip_address=public_ip_address)
assert inference_result, f"Failed to perform inference at IP address: {public_ip_address}"
finally:
ecs_utils.tear_down_ecs_inference_service(ecs_cluster_arn,
service_name, task_family,
revision)
@pytest.mark.skipif(not is_nightly_context(),
reason="Running additional model in nightly context only")
@pytest.mark.model("albert")
@pytest.mark.parametrize("ecs_instance_type", ["c5.4xlarge"], indirect=True)
@pytest.mark.parametrize("ecs_ami", [ECS_AML2_CPU_USWEST2], indirect=True)
def test_ecs_tensorflow_inference_cpu_nlp(tensorflow_inference,
ecs_container_instance, region,
cpu_only):
worker_instance_id, ecs_cluster_arn = ecs_container_instance
public_ip_address = ec2_utils.get_public_ip(worker_instance_id,
region=region)
model_name = "albert"
service_name = task_family = revision = None
try:
service_name, task_family, revision = ecs_utils.setup_ecs_inference_service(
run("kubectl apply -f {}".format(yaml_path))
port_to_forward = random.randint(49152, 65535)
if eks_utils.is_service_running(selector_name):
eks_utils.eks_forward_port_between_host_and_container(selector_name, port_to_forward, "8500")
assert test_utils.request_tensorflow_inference(model_name=model_name, port=port_to_forward)
except ValueError as excp:
eks_utils.LOGGER.error("Service is not running: %s", excp)
finally:
run(f"kubectl delete deployment {selector_name}")
run(f"kubectl delete service {selector_name}")
@pytest.mark.skipif(not test_utils.is_nightly_context(), reason="Running additional model in nightly context only")
@pytest.mark.model("albert")
def test_eks_tensorflow_albert(tensorflow_inference):
if "eia" in tensorflow_inference or "neuron" in tensorflow_inference:
pytest.skip("Skipping EKS Test for EIA and neuron Images")
num_replicas = "1"
rand_int = random.randint(4001, 6000)
processor = "gpu" if "gpu" in tensorflow_inference else "cpu"
model_name = f"albert"
yaml_path = os.path.join(os.sep, "tmp", f"tensorflow_single_node_{processor}_inference_{rand_int}.yaml")
inference_service_name = selector_name = f"albert-{processor}-{rand_int}"
search_replace_dict = {
# TODO: Remove this once we have whitelisted appropriate LOW/MEDIUM vulnerabilities
if not (vulnerability_severity.get("CRITICAL")
or vulnerability_severity.get("HIGH")):
return
raise DependencyCheckFailure(
f"Unrecognized CVEs have been reported : {vulnerability_severity}. "
f"Allowed vulnerabilities are {allowed_vulnerabilities or None}. Please see "
f"{dependency_check_report} for more details.")
@pytest.mark.model("N/A")
@pytest.mark.canary("Run dependency tests regularly on production images")
@pytest.mark.parametrize("ec2_instance_type", ["c5.4xlarge"], indirect=True)
@pytest.mark.skipif(
not (is_nightly_context() or is_mainline_context() or
(is_canary_context() and is_time_for_canary_safety_scan())),
reason="Do not run dependency check on PR tests. "
"Executing test in canaries pipeline during only a limited period of time."
)
def test_dependency_check_cpu(cpu, ec2_connection):
_run_dependency_check_test(cpu, ec2_connection, "cpu")
@pytest.mark.model("N/A")
@pytest.mark.canary("Run dependency tests regularly on production images")
@pytest.mark.parametrize("ec2_instance_type", ["p3.2xlarge"], indirect=True)
@pytest.mark.skipif(
not (is_nightly_context() or is_mainline_context() or
(is_canary_context() and is_time_for_canary_safety_scan())),
reason="Do not run dependency check on PR tests. "
def pytest_generate_tests(metafunc):
images = metafunc.config.getoption("--images")
# Don't parametrize if there are no images to parametrize
if not images:
return
# Parametrize framework specific tests
for fixture in FRAMEWORK_FIXTURES:
if fixture in metafunc.fixturenames:
lookup = fixture.replace("_", "-")
images_to_parametrize = []
for image in images:
if lookup_condition(lookup, image):
is_example_lookup = "example_only" in metafunc.fixturenames and "example" in image
is_huggingface_lookup = (
("huggingface_only" in metafunc.fixturenames
or "huggingface" in metafunc.fixturenames)
and "huggingface" in image)
is_trcomp_lookup = "trcomp" in image and all(
fixture_name not in metafunc.fixturenames
for fixture_name in ["example_only"])
is_standard_lookup = all(
fixture_name not in metafunc.fixturenames
for fixture_name in
["example_only", "huggingface_only"]) and all(
keyword not in image
for keyword in ["example", "huggingface"])
if "sagemaker_only" in metafunc.fixturenames and is_ec2_image(
image):
LOGGER.info(
f"Not running EC2 image {image} on sagemaker_only test"
)
continue
if is_sagemaker_image(image):
if "sagemaker_only" not in metafunc.fixturenames and "sagemaker" not in metafunc.fixturenames:
LOGGER.info(
f"Skipping test, as this function is not marked as 'sagemaker_only' or 'sagemaker'"
)
continue
if not framework_version_within_limit(metafunc, image):
continue
if "non_huggingface_only" in metafunc.fixturenames and "huggingface" in image:
continue
if "non_autogluon_only" in metafunc.fixturenames and "autogluon" in image:
continue
if "x86_compatible_only" in metafunc.fixturenames and "graviton" in image:
continue
if "training_compiler_only" in metafunc.fixturenames and not (
"trcomp" in image):
continue
if is_example_lookup or is_huggingface_lookup or is_standard_lookup or is_trcomp_lookup:
if "cpu_only" in metafunc.fixturenames and "cpu" in image and "eia" not in image:
images_to_parametrize.append(image)
elif "gpu_only" in metafunc.fixturenames and "gpu" in image:
images_to_parametrize.append(image)
elif "graviton_compatible_only" in metafunc.fixturenames and "graviton" in image:
images_to_parametrize.append(image)
elif ("cpu_only" not in metafunc.fixturenames
and "gpu_only" not in metafunc.fixturenames
and "graviton_compatible_only"
not in metafunc.fixturenames):
images_to_parametrize.append(image)
# Remove all images tagged as "py2" if py3_only is a fixture
if images_to_parametrize and "py3_only" in metafunc.fixturenames:
images_to_parametrize = [
py3_image for py3_image in images_to_parametrize
if "py2" not in py3_image
]
if is_nightly_context():
nightly_images_to_parametrize = []
# filter the nightly fixtures in the current functional context
func_nightly_fixtures = {
key: value
for (key, value) in NIGHTLY_FIXTURES.items()
if key in metafunc.fixturenames
}
# iterate through image candidates and select images with labels that match all nightly fixture labels
for image_candidate in images_to_parametrize:
if all([
are_valid_fixture_labels_present(
image_candidate, nightly_labels) for _,
nightly_labels in func_nightly_fixtures.items()
]):
nightly_images_to_parametrize.append(image_candidate)
images_to_parametrize = nightly_images_to_parametrize
# Parametrize tests that spin up an ecs cluster or tests that spin up an EC2 instance with a unique name
values_to_generate_for_fixture = {
"ecs_container_instance": "ecs_cluster_name",
"ec2_connection": "ec2_key_name",
}
fixtures_parametrized = generate_unique_values_for_fixtures(
metafunc, images_to_parametrize,
values_to_generate_for_fixture)
if fixtures_parametrized:
for new_fixture_name, test_parametrization in fixtures_parametrized.items(
):
metafunc.parametrize(f"{fixture},{new_fixture_name}",
test_parametrization)
else:
metafunc.parametrize(fixture, images_to_parametrize)
# Parametrize for framework agnostic tests, i.e. sanity
if "image" in metafunc.fixturenames:
metafunc.parametrize("image", images)