def _run_cmd_on_container(container_name, context, cmd, executable="bash"):
"""
Helper function to run commands on a locally running container
:param container_name: Name of the docker container
:param context: ECR image URI
:param cmd: Command to run on the container
:param executable: Executable to run on the container (bash or python)
:return: invoke output, can be used to parse stdout, etc
"""
if executable not in ("bash", "python"):
LOGGER.warn(
f"Unrecognized executable {executable}. It will be run as {executable} -c '{cmd}'"
)
return context.run(
f"docker exec --user root {container_name} {executable} -c '{cmd}'",
hide=True,
timeout=30)
def test_cuda_paths(gpu):
"""
Test to ensure that:
a. buildspec contains an entry to create the same image as the image URI
b. directory structure for GPU Dockerfiles has framework version, python version, and cuda version in it
:param gpu: gpu image uris
"""
image = gpu
if "example" in image:
pytest.skip(
"Skipping Example Dockerfiles which are not explicitly tied to a cuda version"
)
dlc_path = os.getcwd().split("/test/")[0]
job_type = "training" if "training" in image else "inference"
# Ensure that image has a supported framework
framework, framework_version = get_framework_and_version_from_tag(image)
# Get cuda, framework version, python version through regex
cuda_version = re.search(r"-(cu\d+)-", image).group(1)
framework_short_version = None
python_version = re.search(r"(py\d+)", image).group(1)
short_python_version = None
image_tag = re.search(
r":(\d+(\.\d+){2}(-transformers\d+(\.\d+){2})?-(gpu)-(py\d+)(-cu\d+)-(ubuntu\d+\.\d+)((-e3)?-example|-e3|-sagemaker)?)",
image,
).group(1)
# replacing '_' by '/' to handle huggingface_<framework> case
framework_path = framework.replace("_", "/")
framework_version_path = os.path.join(dlc_path, framework_path, job_type,
"docker", framework_version)
if not os.path.exists(framework_version_path):
framework_short_version = re.match(r"(\d+.\d+)",
framework_version).group(1)
framework_version_path = os.path.join(dlc_path, framework_path,
job_type, "docker",
framework_short_version)
if not os.path.exists(os.path.join(framework_version_path,
python_version)):
# Use the pyX version as opposed to the pyXY version if pyXY path does not exist
short_python_version = python_version[:3]
# Check buildspec for cuda version
buildspec = "buildspec.yml"
if is_tf_version("1", image):
buildspec = "buildspec-tf1.yml"
image_tag_in_buildspec = False
dockerfile_spec_abs_path = None
buildspec_path = os.path.join(dlc_path, framework_path, buildspec)
buildspec_def = Buildspec()
buildspec_def.load(buildspec_path)
for name, image_spec in buildspec_def["images"].items():
if image_spec["device_type"] == "gpu" and image_spec[
"tag"] == image_tag:
image_tag_in_buildspec = True
dockerfile_spec_abs_path = os.path.join(
os.path.dirname(framework_version_path),
image_spec["docker_file"].lstrip("docker/"))
break
try:
assert image_tag_in_buildspec, f"Image tag {image_tag} not found in {buildspec_path}"
except AssertionError as e:
if not is_dlc_cicd_context():
LOGGER.warn(
f"{e} - not failing, as this is a(n) {os.getenv('BUILD_CONTEXT', 'empty')} build context."
)
else:
raise
image_properties_expected_in_dockerfile_path = [
framework_short_version or framework_version,
short_python_version or python_version,
cuda_version,
]
assert all(
prop in dockerfile_spec_abs_path
for prop in image_properties_expected_in_dockerfile_path
), (f"Dockerfile location {dockerfile_spec_abs_path} does not contain all the image properties in "
f"{image_properties_expected_in_dockerfile_path}")
assert os.path.exists(
dockerfile_spec_abs_path
), f"Cannot find dockerfile for {image} in {dockerfile_spec_abs_path}"
def test_cuda_paths(gpu):
"""
Test to ensure that:
a. buildspec contains an entry to create the same image as the image URI
b. directory structure for GPU Dockerfiles has framework version, python version, and cuda version in it
:param gpu: gpu image uris
"""
image = gpu
if "example" in image:
pytest.skip(
"Skipping Example Dockerfiles which are not explicitly tied to a cuda version"
)
dlc_path = os.getcwd().split("/test/")[0]
job_type = "training" if "training" in image else "inference"
# Ensure that image has a supported framework
frameworks = ("tensorflow", "pytorch", "mxnet")
framework = ""
for fw in frameworks:
if fw in image:
framework = fw
break
assert framework, f"Cannot find any frameworks {frameworks} in image uri {image}"
# Get cuda, framework version, python version through regex
cuda_version = re.search(r"-(cu\d+)-", image).group(1)
framework_version = re.search(r":(\d+(\.\d+){2})", image).group(1)
framework_short_version = None
python_version = re.search(r"(py\d+)", image).group(1)
short_python_version = None
image_tag = re.search(
r":(\d+(\.\d+){2}-(cpu|gpu|neuron)-(py\d+)(-cu\d+)-(ubuntu\d+\.\d+)(-example)?)",
image).group(1)
framework_version_path = os.path.join(dlc_path, framework, job_type,
"docker", framework_version)
if not os.path.exists(framework_version_path):
framework_short_version = re.match(r"(\d+.\d+)",
framework_version).group(1)
framework_version_path = os.path.join(dlc_path, framework, job_type,
"docker",
framework_short_version)
if not os.path.exists(os.path.join(framework_version_path,
python_version)):
# Use the pyX version as opposed to the pyXY version if pyXY path does not exist
short_python_version = python_version[:3]
# Check buildspec for cuda version
buildspec = "buildspec.yml"
if is_tf_version("1", image):
buildspec = "buildspec-tf1.yml"
cuda_in_buildspec = False
dockerfile_spec_abs_path = None
cuda_in_buildspec_ref = f"CUDA_VERSION {cuda_version}"
buildspec_path = os.path.join(dlc_path, framework, buildspec)
buildspec_def = Buildspec()
buildspec_def.load(buildspec_path)
for name, image_spec in buildspec_def["images"].items():
if image_spec["device_type"] == "gpu" and image_spec[
"tag"] == image_tag:
cuda_in_buildspec = True
dockerfile_spec_abs_path = os.path.join(
os.path.dirname(framework_version_path),
image_spec["docker_file"].lstrip("docker/"))
break
try:
assert cuda_in_buildspec, f"Can't find {cuda_in_buildspec_ref} in {buildspec_path}"
except AssertionError as e:
if not is_dlc_cicd_context():
LOGGER.warn(
f"{e} - not failing, as this is a(n) {os.getenv('BUILD_CONTEXT', 'empty')} build context."
)
else:
raise
image_properties_expected_in_dockerfile_path = [
framework_short_version or framework_version, short_python_version
or python_version, cuda_version
]
assert all(
prop in dockerfile_spec_abs_path
for prop in image_properties_expected_in_dockerfile_path
), (f"Dockerfile location {dockerfile_spec_abs_path} does not contain all the image properties in "
f"{image_properties_expected_in_dockerfile_path}")
assert os.path.exists(
dockerfile_spec_abs_path
), f"Cannot find dockerfile for {image} in {dockerfile_spec_abs_path}"
def test_cuda_paths(gpu):
"""
Test to ensure directory structure for GPU Dockerfiles has cuda version in it
:param gpu: gpu image uris
"""
image = gpu
if "example" in image:
pytest.skip(
"Skipping Example Dockerfiles which are not explicitly tied to a cuda version"
)
dlc_path = os.getcwd().split("/test/")[0]
job_type = "training" if "training" in image else "inference"
# Ensure that image has a supported framework
frameworks = ("tensorflow", "pytorch", "mxnet")
framework = ""
for fw in frameworks:
if fw in image:
framework = fw
break
assert framework, f"Cannot find any frameworks {frameworks} in image uri {image}"
# Get cuda, framework version, python version through regex
cuda_version = re.search(r"-(cu\d+)-", image).group(1)
framework_version = re.search(r":(\d+(.\d+){2})", image).group(1)
python_version = re.search(r"(py\d+)", image).group(1)
framework_version_path = os.path.join(dlc_path, framework, job_type,
"docker", framework_version)
if not os.path.exists(framework_version_path):
framework_short_version = re.match(r"(\d+.\d+)",
framework_version).group(1)
framework_version_path = os.path.join(dlc_path, framework, job_type,
"docker",
framework_short_version)
if not os.path.exists(os.path.join(framework_version_path,
python_version)):
# Use the pyX version as opposed to the pyXY version if pyXY path does not exist
python_version = python_version[:3]
# Check buildspec for cuda version
buildspec = "buildspec.yml"
if is_tf_version("1", image):
buildspec = "buildspec-tf1.yml"
cuda_in_buildspec = False
cuda_in_buildspec_ref = f"CUDA_VERSION {cuda_version}"
buildspec_path = os.path.join(dlc_path, framework, buildspec)
with open(buildspec_path, "r") as bf:
for line in bf:
if cuda_in_buildspec_ref in line:
cuda_in_buildspec = True
break
try:
assert cuda_in_buildspec, f"Can't find {cuda_in_buildspec_ref} in {buildspec_path}"
except AssertionError as e:
if not is_dlc_cicd_context():
LOGGER.warn(
f"{e} - not failing, as this is a(n) {os.getenv('BUILD_CONTEXT', 'empty')} build context."
)
else:
raise
# Check that a Dockerfile exists in the right directory
dockerfile_path = os.path.join(framework_version_path, python_version,
cuda_version, "Dockerfile.gpu")
assert os.path.exists(
dockerfile_path
), f"Cannot find dockerfile for image {image} in {dockerfile_path}"
