def test_cuda_paths(gpu):
"""
Test to ensure that:
a. buildspec contains an entry to create the same image as the image URI
b. directory structure for GPU Dockerfiles has framework version, python version, and cuda version in it
:param gpu: gpu image uris
"""
image = gpu
if "example" in image:
pytest.skip(
"Skipping Example Dockerfiles which are not explicitly tied to a cuda version"
)
dlc_path = os.getcwd().split("/test/")[0]
job_type = "training" if "training" in image else "inference"
# Ensure that image has a supported framework
framework, framework_version = get_framework_and_version_from_tag(image)
# Get cuda, framework version, python version through regex
cuda_version = re.search(r"-(cu\d+)-", image).group(1)
framework_short_version = None
python_version = re.search(r"(py\d+)", image).group(1)
short_python_version = None
image_tag = re.search(
r":(\d+(\.\d+){2}(-transformers\d+(\.\d+){2})?-(gpu)-(py\d+)(-cu\d+)-(ubuntu\d+\.\d+)((-e3)?-example|-e3|-sagemaker)?)",
image,
).group(1)
# replacing '_' by '/' to handle huggingface_<framework> case
framework_path = framework.replace("_", "/")
framework_version_path = os.path.join(dlc_path, framework_path, job_type,
"docker", framework_version)
if not os.path.exists(framework_version_path):
framework_short_version = re.match(r"(\d+.\d+)",
framework_version).group(1)
framework_version_path = os.path.join(dlc_path, framework_path,
job_type, "docker",
framework_short_version)
if not os.path.exists(os.path.join(framework_version_path,
python_version)):
# Use the pyX version as opposed to the pyXY version if pyXY path does not exist
short_python_version = python_version[:3]
# Check buildspec for cuda version
buildspec = "buildspec.yml"
if is_tf_version("1", image):
buildspec = "buildspec-tf1.yml"
image_tag_in_buildspec = False
dockerfile_spec_abs_path = None
buildspec_path = os.path.join(dlc_path, framework_path, buildspec)
buildspec_def = Buildspec()
buildspec_def.load(buildspec_path)
for name, image_spec in buildspec_def["images"].items():
if image_spec["device_type"] == "gpu" and image_spec[
"tag"] == image_tag:
image_tag_in_buildspec = True
dockerfile_spec_abs_path = os.path.join(
os.path.dirname(framework_version_path),
image_spec["docker_file"].lstrip("docker/"))
break
try:
assert image_tag_in_buildspec, f"Image tag {image_tag} not found in {buildspec_path}"
except AssertionError as e:
if not is_dlc_cicd_context():
LOGGER.warn(
f"{e} - not failing, as this is a(n) {os.getenv('BUILD_CONTEXT', 'empty')} build context."
)
else:
raise
image_properties_expected_in_dockerfile_path = [
framework_short_version or framework_version,
short_python_version or python_version,
cuda_version,
]
assert all(
prop in dockerfile_spec_abs_path
for prop in image_properties_expected_in_dockerfile_path
), (f"Dockerfile location {dockerfile_spec_abs_path} does not contain all the image properties in "
f"{image_properties_expected_in_dockerfile_path}")
assert os.path.exists(
dockerfile_spec_abs_path
), f"Cannot find dockerfile for {image} in {dockerfile_spec_abs_path}"
def test_cuda_paths(gpu):
"""
Test to ensure that:
a. buildspec contains an entry to create the same image as the image URI
b. directory structure for GPU Dockerfiles has framework version, python version, and cuda version in it
:param gpu: gpu image uris
"""
image = gpu
if "example" in image:
pytest.skip(
"Skipping Example Dockerfiles which are not explicitly tied to a cuda version"
)
dlc_path = os.getcwd().split("/test/")[0]
job_type = "training" if "training" in image else "inference"
# Ensure that image has a supported framework
frameworks = ("tensorflow", "pytorch", "mxnet")
framework = ""
for fw in frameworks:
if fw in image:
framework = fw
break
assert framework, f"Cannot find any frameworks {frameworks} in image uri {image}"
# Get cuda, framework version, python version through regex
cuda_version = re.search(r"-(cu\d+)-", image).group(1)
framework_version = re.search(r":(\d+(\.\d+){2})", image).group(1)
framework_short_version = None
python_version = re.search(r"(py\d+)", image).group(1)
short_python_version = None
image_tag = re.search(
r":(\d+(\.\d+){2}-(cpu|gpu|neuron)-(py\d+)(-cu\d+)-(ubuntu\d+\.\d+)(-example)?)",
image).group(1)
framework_version_path = os.path.join(dlc_path, framework, job_type,
"docker", framework_version)
if not os.path.exists(framework_version_path):
framework_short_version = re.match(r"(\d+.\d+)",
framework_version).group(1)
framework_version_path = os.path.join(dlc_path, framework, job_type,
"docker",
framework_short_version)
if not os.path.exists(os.path.join(framework_version_path,
python_version)):
# Use the pyX version as opposed to the pyXY version if pyXY path does not exist
short_python_version = python_version[:3]
# Check buildspec for cuda version
buildspec = "buildspec.yml"
if is_tf_version("1", image):
buildspec = "buildspec-tf1.yml"
cuda_in_buildspec = False
dockerfile_spec_abs_path = None
cuda_in_buildspec_ref = f"CUDA_VERSION {cuda_version}"
buildspec_path = os.path.join(dlc_path, framework, buildspec)
buildspec_def = Buildspec()
buildspec_def.load(buildspec_path)
for name, image_spec in buildspec_def["images"].items():
if image_spec["device_type"] == "gpu" and image_spec[
"tag"] == image_tag:
cuda_in_buildspec = True
dockerfile_spec_abs_path = os.path.join(
os.path.dirname(framework_version_path),
image_spec["docker_file"].lstrip("docker/"))
break
try:
assert cuda_in_buildspec, f"Can't find {cuda_in_buildspec_ref} in {buildspec_path}"
except AssertionError as e:
if not is_dlc_cicd_context():
LOGGER.warn(
f"{e} - not failing, as this is a(n) {os.getenv('BUILD_CONTEXT', 'empty')} build context."
)
else:
raise
image_properties_expected_in_dockerfile_path = [
framework_short_version or framework_version, short_python_version
or python_version, cuda_version
]
assert all(
prop in dockerfile_spec_abs_path
for prop in image_properties_expected_in_dockerfile_path
), (f"Dockerfile location {dockerfile_spec_abs_path} does not contain all the image properties in "
f"{image_properties_expected_in_dockerfile_path}")
assert os.path.exists(
dockerfile_spec_abs_path
), f"Cannot find dockerfile for {image} in {dockerfile_spec_abs_path}"
class DLCReleaseInformation:
def __init__(self, dlc_account_id, dlc_region, dlc_repository, dlc_tag):
if not all([dlc_account_id, dlc_tag, dlc_repository, dlc_region]):
raise ValueError(
"One or multiple environment variables TARGET_ACCOUNT_ID_CLASSIC, TAG_WITH_DLC_VERSION, "
"TARGET_ECR_REPOSITORY, REGION not set. This environment variable is expected to be set by the promoter stage."
)
self.dlc_account_id = dlc_account_id
self.dlc_region = dlc_region
self.dlc_repository = dlc_repository
self.dlc_tag = dlc_tag
self.container_name = self.run_container()
imp_package_list_path = os.path.join(
os.sep, os.path.dirname(__file__), "resources", "important_dlc_packages.yml"
)
self.imp_packages_to_record = Buildspec()
self.imp_packages_to_record.load(imp_package_list_path)
self._image_details = self.get_image_details_from_ecr()
def get_boto3_ecr_client(self):
return boto3.Session(region_name=self.dlc_region).client("ecr")
def run_container(self):
"""
Quickly run a container and assign it a name, so that different commands may be run on it
:return:
"""
container_name = f"{self.dlc_repository}-{self.dlc_tag}-release-information"
run(f"docker rm -f {container_name}", warn=True, hide=True)
run(f"docker run -id --name {container_name} --entrypoint='/bin/bash' {self.image}", hide=True)
return container_name
def get_container_command_output(self, command):
"""
Get stdout of the command executed.
Note: does not handle stderr as it's not important in this context.
:param command:
:return:
"""
docker_exec_cmd = f"docker exec -i {self.container_name}"
run_stdout = run(f"{docker_exec_cmd} {command}", hide=True).stdout.strip()
return run_stdout
def get_image_details_from_ecr(self):
_ecr = self.get_boto3_ecr_client()
try:
response = _ecr.describe_images(
registryId=self.dlc_account_id,
repositoryName=self.dlc_repository,
imageIds=[{"imageTag": self.dlc_tag}],
)
except ClientError as e:
LOGGER.error("ClientError when performing ECR operation. Exception: {}".format(e))
return response["imageDetails"][0]
@property
def image(self):
return f"{self.dlc_account_id}.dkr.ecr.{self.dlc_region}.amazonaws.com/{self.dlc_repository}:{self.dlc_tag}"
@property
def image_tags(self):
return self._image_details["imageTags"]
@property
def image_digest(self):
return self._image_details["imageDigest"]
@property
def bom_pip_packages(self):
return self.get_container_command_output("pip freeze")
@property
def bom_apt_packages(self):
return self.get_container_command_output("apt list --installed")
@property
def bom_pipdeptree(self):
self.get_container_command_output("pip install pipdeptree")
return self.get_container_command_output("pipdeptree")
@property
def imp_pip_packages(self):
imp_pip_packages = {}
container_pip_packages = json.loads(self.get_container_command_output("pip list --format=json"))
for pip_package in sorted(self.imp_packages_to_record["pip_packages"]):
for package_entry in container_pip_packages:
if package_entry["name"].lower() == pip_package.lower():
imp_pip_packages[package_entry["name"]] = package_entry["version"]
break
return imp_pip_packages
@property
def imp_apt_packages(self):
imp_apt_packages = []
for apt_package in sorted(self.imp_packages_to_record["apt_packages"]):
apt_package_name = self.get_container_command_output(
f"dpkg --get-selections | grep -i {apt_package} | awk '{{print $1}}'"
)
if apt_package_name:
imp_apt_packages.append(apt_package_name.replace("\n", " & "))
return imp_apt_packages