def test_ubuntu_version(image):
"""
Check that the ubuntu version in the image tag is the same as the one on a running container.
:param image: ECR image URI
"""
ctx = Context()
container_name = get_container_name("ubuntu-version", image)
ubuntu_version = ""
for tag_split in image.split("-"):
if tag_split.startswith("ubuntu"):
ubuntu_version = tag_split.split("ubuntu")[-1]
start_container(container_name, image, ctx)
output = run_cmd_on_container(container_name, ctx, "cat /etc/os-release")
container_ubuntu_version = output.stdout
assert "Ubuntu" in container_ubuntu_version
assert ubuntu_version in container_ubuntu_version
def test_tf_serving_version_cpu(tensorflow_inference):
"""
For non-huggingface non-GPU TF inference images, check that the tag version matches the version of TF serving
in the container.
Huggingface includes MMS and core TF, hence the versioning scheme is based off of the underlying tensorflow
framework version, rather than the TF serving version.
GPU inference images will be tested along side `test_framework_and_cuda_version_gpu` in order to be judicious
about GPU resources. This test can run directly on the host, and thus does not require additional resources
to be spun up.
@param tensorflow_inference: ECR image URI
"""
# Set local variable to clarify contents of fixture
image = tensorflow_inference
if "gpu" in image:
pytest.skip(
"GPU images will have their framework version tested in test_framework_and_cuda_version_gpu"
)
if "neuron" in image:
pytest.skip(
"Neuron images will have their framework version tested in test_framework_and_neuron_sdk_version"
)
_, tag_framework_version = get_framework_and_version_from_tag(image)
ctx = Context()
container_name = get_container_name("tf-serving-version", image)
start_container(container_name, image, ctx)
output = run_cmd_on_container(container_name,
ctx,
"tensorflow_model_server --version",
executable="bash")
assert re.match(rf"TensorFlow ModelServer: {tag_framework_version}(\D+)?", output.stdout), \
f"Cannot find model server version {tag_framework_version} in {output.stdout}"
stop_and_remove_container(container_name, ctx)
def test_framework_version_cpu(image):
"""
Check that the framework version in the image tag is the same as the one on a running container.
This function tests CPU, EIA, and Neuron images.
:param image: ECR image URI
"""
if "gpu" in image:
pytest.skip(
"GPU images will have their framework version tested in test_framework_and_cuda_version_gpu"
)
image_repo_name, _ = get_repository_and_tag_from_image_uri(image)
if re.fullmatch(r"(pr-|beta-|nightly-)?tensorflow-inference(-eia)?",
image_repo_name):
pytest.skip(
msg=
"TF inference for CPU/GPU/EIA does not have core tensorflow installed"
)
tested_framework, tag_framework_version = get_framework_and_version_from_tag(
image)
# Framework name may include huggingface
tested_framework = tested_framework.lstrip("huggingface_")
# Module name is torch
if tested_framework == "pytorch":
tested_framework = "torch"
ctx = Context()
container_name = get_container_name("framework-version", image)
start_container(container_name, image, ctx)
output = run_cmd_on_container(
container_name,
ctx,
f"import {tested_framework}; print({tested_framework}.__version__)",
executable="python")
if is_canary_context():
assert tag_framework_version in output.stdout.strip()
else:
assert tag_framework_version == output.stdout.strip()
def test_python_version(image):
"""
Check that the python version in the image tag is the same as the one on a running container.
:param image: ECR image URI
"""
ctx = Context()
container_name = get_container_name("py-version", image)
py_version = ""
for tag_split in image.split("-"):
if tag_split.startswith("py"):
if len(tag_split) > 3:
py_version = f"Python {tag_split[2]}.{tag_split[3]}"
else:
py_version = f"Python {tag_split[2]}"
start_container(container_name, image, ctx)
output = run_cmd_on_container(container_name, ctx, "python --version")
# Due to py2 deprecation, Python2 version gets streamed to stderr. Python installed via Conda also appears to
# stream to stderr (in some cases).
container_py_version = output.stdout + output.stderr
assert py_version in container_py_version, f"Cannot find {py_version} in {container_py_version}"
def test_utility_packages_using_import(training):
"""
Verify that utility packages are installed in the Training DLC image
:param training: training ECR image URI
"""
ctx = Context()
container_name = test_utils.get_container_name(
"utility_packages_using_import", training)
test_utils.start_container(container_name, training, ctx)
framework, framework_version = test_utils.get_framework_and_version_from_tag(
training)
utility_package_minimum_framework_version = {
"mxnet": "1.8",
"pytorch": "1.7",
"tensorflow2": "2.4",
"tensorflow1": "1.15",
}
framework = "tensorflow1" if framework == "tensorflow" and framework_version.startswith(
"1.") else "tensorflow2"
if Version(framework_version) < Version(
utility_package_minimum_framework_version[framework]):
pytest.skip("Extra utility packages will be added going forward.")
for package in UTILITY_PACKAGES_IMPORT:
version = test_utils.run_cmd_on_container(
container_name,
ctx,
f"import {package}; print({package}.__version__)",
executable="python").stdout.strip()
if package == "sagemaker":
assert Version(version) > Version(
"2"
), f"Sagemaker version should be > 2.0. Found version {sm_version}"
def test_oss_compliance(image):
"""
Run oss compliance check on a container to check if license attribution files exist.
And upload source of third party packages to S3 bucket.
"""
THIRD_PARTY_SOURCE_CODE_BUCKET = "aws-dlinfra-licenses"
THIRD_PARTY_SOURCE_CODE_BUCKET_PATH = "third_party_source_code"
file = "THIRD_PARTY_SOURCE_CODE_URLS"
container_name = get_container_name("oss_compliance", image)
context = Context()
local_repo_path = get_repository_local_path()
start_container(container_name, image, context)
# run compliance test to make sure license attribution files exists. testOSSCompliance is copied as part of Dockerfile
run_cmd_on_container(container_name, context,
"/usr/local/bin/testOSSCompliance /root")
try:
context.run(
f"docker cp {container_name}:/root/{file} {os.path.join(local_repo_path, file)}"
)
finally:
context.run(f"docker rm -f {container_name}", hide=True)
s3_resource = boto3.resource("s3")
with open(os.path.join(local_repo_path, file)) as source_code_file:
for line in source_code_file:
name, version, url = line.split(" ")
file_name = f"{name}_v{version}_source_code"
s3_object_path = f"{THIRD_PARTY_SOURCE_CODE_BUCKET_PATH}/{file_name}.tar.gz"
local_file_path = os.path.join(local_repo_path, file_name)
for i in range(3):
try:
if not os.path.isdir(local_file_path):
context.run(
f"git clone {url.rstrip()} {local_file_path}")
context.run(
f"tar -czvf {local_file_path}.tar.gz {local_file_path}"
)
except Exception as e:
time.sleep(1)
if i == 2:
LOGGER.error(f"Unable to clone git repo. Error: {e}")
raise
continue
try:
if os.path.exists(f"{local_file_path}.tar.gz"):
LOGGER.info(f"Uploading package to s3 bucket: {line}")
s3_resource.Object(THIRD_PARTY_SOURCE_CODE_BUCKET,
s3_object_path).load()
except botocore.exceptions.ClientError as e:
if e.response["Error"]["Code"] == "404":
try:
# using aws cli as using boto3 expects to upload folder by iterating through each file instead of entire folder.
context.run(
f"aws s3 cp {local_file_path}.tar.gz s3://{THIRD_PARTY_SOURCE_CODE_BUCKET}/{s3_object_path}"
)
object = s3_resource.Bucket(
THIRD_PARTY_SOURCE_CODE_BUCKET).Object(
s3_object_path)
object.Acl().put(ACL="public-read")
except ClientError as e:
LOGGER.error(
f"Unable to upload source code to bucket {THIRD_PARTY_SOURCE_CODE_BUCKET}. Error: {e}"
)
raise
else:
LOGGER.error(
f"Unable to check if source code is present on bucket {THIRD_PARTY_SOURCE_CODE_BUCKET}. Error: {e}"
)
raise
def test_framework_version_cpu(image):
"""
Check that the framework version in the image tag is the same as the one on a running container.
This function tests CPU, EIA images.
:param image: ECR image URI
"""
if "gpu" in image:
pytest.skip(
"GPU images will have their framework version tested in test_framework_and_cuda_version_gpu"
)
if "neuron" in image:
pytest.skip(
"Neuron images will have their framework version tested in test_framework_and_neuron_sdk_version"
)
image_repo_name, _ = get_repository_and_tag_from_image_uri(image)
if re.fullmatch(
r"(pr-|beta-|nightly-)?tensorflow-inference(-eia|-graviton)?",
image_repo_name):
pytest.skip(
"Non-gpu tensorflow-inference images will be tested in test_tf_serving_version_cpu."
)
tested_framework, tag_framework_version = get_framework_and_version_from_tag(
image)
# Framework name may include huggingface
if tested_framework.startswith('huggingface_'):
tested_framework = tested_framework[len("huggingface_"):]
# Module name is torch
if tested_framework == "pytorch":
tested_framework = "torch"
elif tested_framework == "autogluon":
tested_framework = "autogluon.core"
ctx = Context()
container_name = get_container_name("framework-version", image)
start_container(container_name, image, ctx)
output = run_cmd_on_container(
container_name,
ctx,
f"import {tested_framework}; print({tested_framework}.__version__)",
executable="python")
if is_canary_context():
assert tag_framework_version in output.stdout.strip()
else:
if tested_framework == "autogluon.core":
version_to_check = "0.3.1" if tag_framework_version == "0.3.2" else tag_framework_version
assert output.stdout.strip().startswith(version_to_check)
# Habana v1.2 binary does not follow the X.Y.Z+cpu naming convention
elif "habana" not in image_repo_name:
if tested_framework == "torch" and Version(
tag_framework_version) >= Version("1.10.0"):
torch_version_pattern = r"{torch_version}(\+cpu)".format(
torch_version=tag_framework_version)
assert re.fullmatch(
torch_version_pattern, output.stdout.strip()
), (f"torch.__version__ = {output.stdout.strip()} does not match {torch_version_pattern}\n"
f"Please specify framework version as X.Y.Z+cpu")
else:
if "neuron" in image:
assert tag_framework_version in output.stdout.strip()
if all(_string in image
for _string in ["pytorch", "habana", "synapseai1.3.0"]):
# Habana Pytorch version looks like 1.10.0a0+gitb488e78 for SynapseAI1.3 PT1.10.1 images
pt_fw_version_pattern = r"(\d+(\.\d+){1,2}(-rc\d)?)((a0\+git\w{7}))"
pt_fw_version_match = re.fullmatch(pt_fw_version_pattern,
output.stdout.strip())
# This is desired for PT1.10.1 images
assert pt_fw_version_match.group(1) == "1.10.0"
else:
assert tag_framework_version == output.stdout.strip()
stop_and_remove_container(container_name, ctx)
def test_framework_version_cpu(image):
"""
Check that the framework version in the image tag is the same as the one on a running container.
This function tests CPU, EIA images.
:param image: ECR image URI
"""
if "gpu" in image:
pytest.skip(
"GPU images will have their framework version tested in test_framework_and_cuda_version_gpu"
)
if "neuron" in image:
pytest.skip(
"Neuron images will have their framework version tested in test_framework_and_neuron_sdk_version"
)
image_repo_name, _ = get_repository_and_tag_from_image_uri(image)
if re.fullmatch(
r"(pr-|beta-|nightly-)?tensorflow-inference(-eia|-graviton)?",
image_repo_name):
pytest.skip(
msg=
"TF inference for CPU/GPU/EIA does not have core tensorflow installed"
)
tested_framework, tag_framework_version = get_framework_and_version_from_tag(
image)
# Framework name may include huggingface
if tested_framework.startswith('huggingface_'):
tested_framework = tested_framework[len("huggingface_"):]
# Module name is torch
if tested_framework == "pytorch":
tested_framework = "torch"
elif tested_framework == "autogluon":
tested_framework = "autogluon.core"
ctx = Context()
container_name = get_container_name("framework-version", image)
start_container(container_name, image, ctx)
output = run_cmd_on_container(
container_name,
ctx,
f"import {tested_framework}; print({tested_framework}.__version__)",
executable="python")
if is_canary_context():
assert tag_framework_version in output.stdout.strip()
else:
if tested_framework == "autogluon.core":
assert output.stdout.strip().startswith(tag_framework_version)
# Habana v1.2 binary does not follow the X.Y.Z+cpu naming convention
elif "habana" not in image_repo_name:
if tested_framework == "torch" and Version(
tag_framework_version) >= Version("1.10.0"):
torch_version_pattern = r"{torch_version}(\+cpu)".format(
torch_version=tag_framework_version)
assert re.fullmatch(
torch_version_pattern, output.stdout.strip()
), (f"torch.__version__ = {output.stdout.strip()} does not match {torch_version_pattern}\n"
f"Please specify framework version as X.Y.Z+cpu")
else:
if "neuron" in image:
assert tag_framework_version in output.stdout.strip()
else:
assert tag_framework_version == output.stdout.strip()
stop_and_remove_container(container_name, ctx)