def pytest_generate_tests(metafunc):
images = metafunc.config.getoption("--images")
# Don't parametrize if there are no images to parametrize
if not images:
return
# Parametrize framework specific tests
for fixture in FRAMEWORK_FIXTURES:
if fixture in metafunc.fixturenames:
lookup = fixture.replace("_", "-")
images_to_parametrize = []
for image in images:
if lookup_condition(lookup, image):
is_example_lookup = "example_only" in metafunc.fixturenames and "example" in image
is_huggingface_lookup = (
"huggingface_only" in metafunc.fixturenames
or "huggingface"
in metafunc.fixturenames) and "huggingface" in image
is_standard_lookup = all(
fixture_name not in metafunc.fixturenames
for fixture_name in
["example_only", "huggingface_only"]) and all(
keyword not in image
for keyword in ["example", "huggingface"])
if "sagemaker_only" in metafunc.fixturenames and is_e3_image(
image):
LOGGER.info(
f"Not running E3 image {image} on sagemaker_only test"
)
continue
if is_sagemaker_image(image):
if "sagemaker_only" not in metafunc.fixturenames and "sagemaker" not in metafunc.fixturenames:
LOGGER.info(
f"Skipping test, as this function is not marked as 'sagemaker_only' or 'sagemaker'"
)
continue
if not framework_version_within_limit(metafunc, image):
continue
if "non_huggingface_only" in metafunc.fixturenames and "huggingface" in image:
continue
if "non_autogluon_only" in metafunc.fixturenames and "autogluon" in image:
continue
if "x86_compatible_only" in metafunc.fixturenames and "graviton" in image:
continue
if is_example_lookup or is_huggingface_lookup or is_standard_lookup:
if "cpu_only" in metafunc.fixturenames and "cpu" in image and "eia" not in image:
images_to_parametrize.append(image)
elif "gpu_only" in metafunc.fixturenames and "gpu" in image:
images_to_parametrize.append(image)
elif "graviton_compatible_only" in metafunc.fixturenames and "graviton" in image:
images_to_parametrize.append(image)
elif ("cpu_only" not in metafunc.fixturenames
and "gpu_only" not in metafunc.fixturenames
and "graviton_compatible_only"
not in metafunc.fixturenames):
images_to_parametrize.append(image)
# Remove all images tagged as "py2" if py3_only is a fixture
if images_to_parametrize and "py3_only" in metafunc.fixturenames:
images_to_parametrize = [
py3_image for py3_image in images_to_parametrize
if "py2" not in py3_image
]
# Parametrize tests that spin up an ecs cluster or tests that spin up an EC2 instance with a unique name
values_to_generate_for_fixture = {
"ecs_container_instance": "ecs_cluster_name",
"ec2_connection": "ec2_key_name",
}
fixtures_parametrized = generate_unique_values_for_fixtures(
metafunc, images_to_parametrize,
values_to_generate_for_fixture)
if fixtures_parametrized:
for new_fixture_name, test_parametrization in fixtures_parametrized.items(
):
metafunc.parametrize(f"{fixture},{new_fixture_name}",
test_parametrization)
else:
metafunc.parametrize(fixture, images_to_parametrize)
# Parametrize for framework agnostic tests, i.e. sanity
if "image" in metafunc.fixturenames:
metafunc.parametrize("image", images)
def pytest_generate_tests(metafunc):
images = metafunc.config.getoption("--images")
# Don't parametrize if there are no images to parametrize
if not images:
return
# Parametrize framework specific tests
for fixture in FRAMEWORK_FIXTURES:
if fixture in metafunc.fixturenames:
lookup = fixture.replace("_", "-")
images_to_parametrize = []
for image in images:
if lookup_condition(lookup, image):
is_example_lookup = "example_only" in metafunc.fixturenames and "example" in image
is_huggingface_lookup = (
("huggingface_only" in metafunc.fixturenames
or "huggingface" in metafunc.fixturenames)
and "huggingface" in image)
is_trcomp_lookup = "trcomp" in image and all(
fixture_name not in metafunc.fixturenames
for fixture_name in ["example_only"])
is_standard_lookup = all(
fixture_name not in metafunc.fixturenames
for fixture_name in
["example_only", "huggingface_only"]) and all(
keyword not in image
for keyword in ["example", "huggingface"])
if "sagemaker_only" in metafunc.fixturenames and is_ec2_image(
image):
LOGGER.info(
f"Not running EC2 image {image} on sagemaker_only test"
)
continue
if is_sagemaker_image(image):
if "sagemaker_only" not in metafunc.fixturenames and "sagemaker" not in metafunc.fixturenames:
LOGGER.info(
f"Skipping test, as this function is not marked as 'sagemaker_only' or 'sagemaker'"
)
continue
if not framework_version_within_limit(metafunc, image):
continue
if "non_huggingface_only" in metafunc.fixturenames and "huggingface" in image:
continue
if "non_autogluon_only" in metafunc.fixturenames and "autogluon" in image:
continue
if "x86_compatible_only" in metafunc.fixturenames and "graviton" in image:
continue
if "training_compiler_only" in metafunc.fixturenames and not (
"trcomp" in image):
continue
if is_example_lookup or is_huggingface_lookup or is_standard_lookup or is_trcomp_lookup:
if "cpu_only" in metafunc.fixturenames and "cpu" in image and "eia" not in image:
images_to_parametrize.append(image)
elif "gpu_only" in metafunc.fixturenames and "gpu" in image:
images_to_parametrize.append(image)
elif "graviton_compatible_only" in metafunc.fixturenames and "graviton" in image:
images_to_parametrize.append(image)
elif ("cpu_only" not in metafunc.fixturenames
and "gpu_only" not in metafunc.fixturenames
and "graviton_compatible_only"
not in metafunc.fixturenames):
images_to_parametrize.append(image)
# Remove all images tagged as "py2" if py3_only is a fixture
if images_to_parametrize and "py3_only" in metafunc.fixturenames:
images_to_parametrize = [
py3_image for py3_image in images_to_parametrize
if "py2" not in py3_image
]
if is_nightly_context():
nightly_images_to_parametrize = []
# filter the nightly fixtures in the current functional context
func_nightly_fixtures = {
key: value
for (key, value) in NIGHTLY_FIXTURES.items()
if key in metafunc.fixturenames
}
# iterate through image candidates and select images with labels that match all nightly fixture labels
for image_candidate in images_to_parametrize:
if all([
are_valid_fixture_labels_present(
image_candidate, nightly_labels) for _,
nightly_labels in func_nightly_fixtures.items()
]):
nightly_images_to_parametrize.append(image_candidate)
images_to_parametrize = nightly_images_to_parametrize
# Parametrize tests that spin up an ecs cluster or tests that spin up an EC2 instance with a unique name
values_to_generate_for_fixture = {
"ecs_container_instance": "ecs_cluster_name",
"ec2_connection": "ec2_key_name",
}
fixtures_parametrized = generate_unique_values_for_fixtures(
metafunc, images_to_parametrize,
values_to_generate_for_fixture)
if fixtures_parametrized:
for new_fixture_name, test_parametrization in fixtures_parametrized.items(
):
metafunc.parametrize(f"{fixture},{new_fixture_name}",
test_parametrization)
else:
metafunc.parametrize(fixture, images_to_parametrize)
# Parametrize for framework agnostic tests, i.e. sanity
if "image" in metafunc.fixturenames:
metafunc.parametrize("image", images)