def setup_sm_benchmark_tf_train_env(resources_location, setup_tf1_env,
setup_tf2_env):
"""
Create a virtual environment for benchmark tests if it doesn't already exist, and download all necessary scripts
:param resources_location: <str> directory in which test resources should be placed
:param setup_tf1_env: <bool> True if tf1 resources need to be setup
:param setup_tf2_env: <bool> True if tf2 resources need to be setup
:return: absolute path to the location of the virtual environment
"""
ctx = Context()
tf_resource_dir_list = []
if setup_tf1_env:
tf_resource_dir_list.append("tensorflow1")
if setup_tf2_env:
tf_resource_dir_list.append("tensorflow2")
for resource_dir in tf_resource_dir_list:
with ctx.cd(os.path.join(resources_location, resource_dir)):
if not os.path.isdir(
os.path.join(resources_location, resource_dir, "horovod")):
# v0.19.4 is the last version for which horovod example tests are py2 compatible
ctx.run(
"git clone -b v0.19.4 https://github.com/horovod/horovod.git"
)
if not os.path.isdir(
os.path.join(resources_location, resource_dir,
"deep-learning-models")):
# We clone branch tf2 for both 1.x and 2.x tests because tf2 branch contains all necessary files
ctx.run(
f"git clone -b tf2 https://github.com/aws-samples/deep-learning-models.git"
)
venv_dir = os.path.join(resources_location, "sm_benchmark_venv")
if not os.path.isdir(venv_dir):
ctx.run(f"virtualenv {venv_dir}")
with ctx.prefix(f"source {venv_dir}/bin/activate"):
ctx.run(
"pip install 'sagemaker>=2,<3' awscli boto3 botocore six==1.11"
)
# SageMaker TF estimator is coded to only accept framework versions up to 2.1.0 as py2 compatible.
# Fixing this through the following changes:
estimator_location = ctx.run(
"echo $(pip3 show sagemaker |grep 'Location' |sed s/'Location: '//g)/sagemaker/tensorflow/estimator.py"
).stdout.strip("\n")
system = ctx.run("uname -s").stdout.strip("\n")
sed_input_arg = "'' " if system == "Darwin" else ""
ctx.run(
f"sed -i {sed_input_arg}'s/\[2, 1, 0\]/\[2, 1, 1\]/g' {estimator_location}"
)
return venv_dir
def daemon_runner(pytestconfig, data_dir, downloads_dir, working_dir):
"""
Provide an invoke's `Local` object that has started the arduino-cli in daemon mode.
This way is simple to start and kill the daemon when the test is finished
via the kill() function
Useful reference:
http://docs.pyinvoke.org/en/1.4/api/runners.html#invoke.runners.Local
http://docs.pyinvoke.org/en/1.4/api/runners.html
"""
cli_full_line = str(Path(pytestconfig.rootdir).parent / "arduino-cli daemon")
env = {
"ARDUINO_DATA_DIR": data_dir,
"ARDUINO_DOWNLOADS_DIR": downloads_dir,
"ARDUINO_SKETCHBOOK_DIR": data_dir,
}
(Path(data_dir) / "packages").mkdir()
run_context = Context()
# It might happen that we need to change directories between drives on Windows,
# in that case the "/d" flag must be used otherwise directory wouldn't change
cd_command = "cd"
if platform.system() == "Windows":
cd_command += " /d"
# Context.cd() is not used since it doesn't work correctly on Windows.
# It escapes spaces in the path using "\ " but it doesn't always work,
# wrapping the path in quotation marks is the safest approach
run_context.prefix(f'{cd_command} "{working_dir}"')
# Local Class is the implementation of a Runner abstract class
runner = Local(run_context)
runner.run(cli_full_line, echo=False, hide=True, warn=True, env=env, asynchronous=True)
# we block here until the test function using this fixture has returned
yield runner
# Kill the runner's process as we finished our test (platform dependent)
os_signal = signal.SIGTERM
if platform.system() != "Windows":
os_signal = signal.SIGKILL
os.kill(runner.process.pid, os_signal)
def test_sm_profiler_pt(pytorch_training):
processor = get_processor_from_image_uri(pytorch_training)
if processor not in ("cpu", "gpu"):
pytest.skip(f"Processor {processor} not supported. Skipping test.")
_, image_framework_version = get_framework_and_version_from_tag(pytorch_training)
if Version(image_framework_version) in SpecifierSet(">=1.12"):
pytest.skip("sm profiler ZCC test is not supported in PT 1.12 and above")
ctx = Context()
profiler_tests_dir = os.path.join(
os.getenv("CODEBUILD_SRC_DIR"), get_container_name("smprof", pytorch_training), "smprofiler_tests"
)
ctx.run(f"mkdir -p {profiler_tests_dir}", hide=True)
# Download sagemaker-tests zip
sm_tests_zip = "sagemaker-tests.zip"
ctx.run(
f"aws s3 cp {os.getenv('SMPROFILER_TESTS_BUCKET')}/{sm_tests_zip} {profiler_tests_dir}/{sm_tests_zip}",
hide=True,
)
# PT test setup requirements
with ctx.prefix(f"cd {profiler_tests_dir}"):
ctx.run(f"unzip {sm_tests_zip}", hide=True)
with ctx.prefix("cd sagemaker-tests/tests/scripts/pytorch_scripts"):
ctx.run("mkdir -p data", hide=True)
ctx.run(
"aws s3 cp s3://smdebug-testing/datasets/cifar-10-python.tar.gz data/cifar-10-batches-py.tar.gz",
hide=True,
)
ctx.run("aws s3 cp s3://smdebug-testing/datasets/MNIST_pytorch.tar.gz data/MNIST_pytorch.tar.gz", hide=True)
with ctx.prefix("cd data"):
ctx.run("tar -zxf MNIST_pytorch.tar.gz", hide=True)
ctx.run("tar -zxf cifar-10-batches-py.tar.gz", hide=True)
run_sm_profiler_tests(pytorch_training, profiler_tests_dir, "test_profiler_pytorch.py", processor)
def setup_sm_benchmark_mx_train_env(resources_location):
"""
Create a virtual environment for benchmark tests if it doesn't already exist, and download all necessary scripts
:param resources_location: <str> directory in which test resources should be placed
:return: absolute path to the location of the virtual environment
"""
ctx = Context()
venv_dir = os.path.join(resources_location, "sm_benchmark_venv")
if not os.path.isdir(venv_dir):
ctx.run(f"virtualenv {venv_dir}")
with ctx.prefix(f"source {venv_dir}/bin/activate"):
ctx.run("pip install -U 'sagemaker<2' awscli boto3 botocore")
return venv_dir
def execute_sagemaker_remote_tests(image):
"""
Run pytest in a virtual env for a particular image
Expected to run via multiprocessing
:param image: ECR url
"""
pytest_command, path, tag, job_type = generate_sagemaker_pytest_cmd(
image, SAGEMAKER_REMOTE_TEST_TYPE)
context = Context()
with context.cd(path):
context.run(f"virtualenv {tag}")
with context.prefix(f"source {tag}/bin/activate"):
context.run("pip install -r requirements.txt", warn=True)
res = context.run(pytest_command, warn=True)
metrics_utils.send_test_result_metrics(res.return_code)
def run_sagemaker_pytest_cmd(image):
"""
Run pytest in a virtual env for a particular image
Expected to run via multiprocessing
:param image: ECR url
"""
pytest_command, path, tag = generate_sagemaker_pytest_cmd(image)
context = Context()
with context.cd(path):
context.run(f"virtualenv {tag}")
with context.prefix(f"source {tag}/bin/activate"):
context.run("pip install -r requirements.txt", warn=True)
context.run(pytest_command)
def _run(cmd_string, custom_working_dir=None, custom_env=None):
if not custom_working_dir:
custom_working_dir = working_dir
if not custom_env:
custom_env = env
cli_full_line = '"{}" {}'.format(cli_path, cmd_string)
run_context = Context()
# It might happen that we need to change directories between drives on Windows,
# in that case the "/d" flag must be used otherwise directory wouldn't change
cd_command = "cd"
if platform.system() == "Windows":
cd_command += " /d"
# Context.cd() is not used since it doesn't work correctly on Windows.
# It escapes spaces in the path using "\ " but it doesn't always work,
# wrapping the path in quotation marks is the safest approach
with run_context.prefix(f'{cd_command} "{custom_working_dir}"'):
return run_context.run(cli_full_line, echo=False, hide=True, warn=True, env=custom_env, encoding="utf-8")
def execute_sagemaker_remote_tests(process_index, image, global_pytest_cache,
pytest_cache_params):
"""
Run pytest in a virtual env for a particular image. Creates a custom directory for each thread for pytest cache file.
Stores pytest cache in a shared dict.
Expected to run via multiprocessing
:param process_index - id for process. Used to create a custom cache dir
:param image - ECR url
:param global_pytest_cache - shared Manager().dict() for cache merging
:param pytest_cache_params - parameters required for s3 file path building
"""
account_id = os.getenv(
"ACCOUNT_ID",
boto3.client("sts").get_caller_identity()["Account"])
pytest_cache_util = PytestCache(boto3.client("s3"), account_id)
pytest_command, path, tag, job_type = generate_sagemaker_pytest_cmd(
image, SAGEMAKER_REMOTE_TEST_TYPE)
context = Context()
with context.cd(path):
context.run(f"virtualenv {tag}")
with context.prefix(f"source {tag}/bin/activate"):
context.run("pip install -r requirements.txt", warn=True)
pytest_cache_util.download_pytest_cache_from_s3_to_local(
path,
**pytest_cache_params,
custom_cache_directory=str(process_index))
# adding -o cache_dir with a custom directory name
pytest_command += f" -o cache_dir={os.path.join(str(process_index), '.pytest_cache')}"
res = context.run(pytest_command, warn=True)
metrics_utils.send_test_result_metrics(res.return_code)
cache_json = pytest_cache_util.convert_pytest_cache_file_to_json(
path, custom_cache_directory=str(process_index))
global_pytest_cache.update(cache_json)
if res.failed:
raise DLCSageMakerRemoteTestFailure(
f"{pytest_command} failed with error code: {res.return_code}\n"
f"Traceback:\n{res.stdout}")
return None
def setup_sm_benchmark_tf_train_env(resources_location, setup_tf1_env,
setup_tf2_env):
"""
Create a virtual environment for benchmark tests if it doesn't already exist, and download all necessary scripts
:param resources_location: <str> directory in which test resources should be placed
:param setup_tf1_env: <bool> True if tf1 resources need to be setup
:param setup_tf2_env: <bool> True if tf2 resources need to be setup
:return: absolute path to the location of the virtual environment
"""
ctx = Context()
tf_resource_dir_list = []
if setup_tf1_env:
tf_resource_dir_list.append("tensorflow1")
if setup_tf2_env:
tf_resource_dir_list.append("tensorflow2")
for resource_dir in tf_resource_dir_list:
with ctx.cd(os.path.join(resources_location, resource_dir)):
if not os.path.isdir(
os.path.join(resources_location, resource_dir, "horovod")):
ctx.run("git clone https://github.com/horovod/horovod.git")
if not os.path.isdir(
os.path.join(resources_location, resource_dir,
"deep-learning-models")):
# We clone branch tf2 for both 1.x and 2.x tests because tf2 branch contains all necessary files
ctx.run(
f"git clone -b tf2 https://github.com/aws-samples/deep-learning-models.git"
)
venv_dir = os.path.join(resources_location, "sm_benchmark_venv")
if not os.path.isdir(venv_dir):
ctx.run(f"virtualenv {venv_dir}")
with ctx.prefix(f"source {venv_dir}/bin/activate"):
ctx.run("pip install -U sagemaker awscli boto3 botocore six==1.11")
return venv_dir
def run_sm_perf_test(image_uri, num_nodes, region):
"""
Run TF sagemaker training performance tests
Additional context: Setup for this function is performed by 'setup_sm_benchmark_tf_train_env' -- this installs
some prerequisite packages, clones some repos, and creates a virtualenv called sm_benchmark_venv.
TODO: Refactor the above setup function to be more obviously connected to this function,
TODO: and install requirements via a requirements.txt file
:param image_uri: ECR image URI
:param num_nodes: Number of nodes to run on
:param region: AWS region
"""
_, framework_version = get_framework_and_version_from_tag(image_uri)
if framework_version.startswith("1."):
pytest.skip("Skipping benchmark test on TF 1.x images.")
processor = "gpu" if "gpu" in image_uri else "cpu"
device_cuda_str = f"{processor}-{get_cuda_version_from_tag(image_uri)}" if processor == "gpu" else processor
ec2_instance_type = "p3.16xlarge" if processor == "gpu" else "c5.18xlarge"
py_version = "py2" if "py2" in image_uri else "py37" if "py37" in image_uri else "py3"
time_str = time.strftime("%Y-%m-%d-%H-%M-%S")
commit_info = os.getenv("CODEBUILD_RESOLVED_SOURCE_VERSION")
target_upload_location = os.path.join(BENCHMARK_RESULTS_S3_BUCKET,
"tensorflow", framework_version,
"sagemaker", "training",
device_cuda_str, py_version)
training_job_name = (
f"tf{framework_version[0]}-tr-bench-{device_cuda_str}-{num_nodes}-node-{py_version}-{commit_info[:7]}-{time_str}"
)
# Inserting random sleep because this test starts multiple training jobs around the same time, resulting in
# a throttling error for SageMaker APIs.
time.sleep(Random(x=training_job_name).random() * 60)
test_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),
"resources")
venv_dir = os.path.join(test_dir, "sm_benchmark_venv")
ctx = Context()
with ctx.cd(test_dir), ctx.prefix(f"source {venv_dir}/bin/activate"):
log_file = (
f"results-{commit_info}-{time_str}-{framework_version}-{device_cuda_str}-{py_version}-{num_nodes}-node.txt"
)
run_out = ctx.run(
f"timeout 45m python tf_sm_benchmark.py "
f"--framework-version {framework_version} "
f"--image-uri {image_uri} "
f"--instance-type ml.{ec2_instance_type} "
f"--node-count {num_nodes} "
f"--python {py_version} "
f"--region {region} "
f"--job-name {training_job_name}"
f"2>&1 | tee {log_file}",
warn=True,
echo=True,
)
if not (run_out.ok or run_out.return_code == 124):
target_upload_location = os.path.join(target_upload_location,
"failure_log")
ctx.run(
f"aws s3 cp {os.path.join(test_dir, log_file)} {os.path.join(target_upload_location, log_file)}"
)
LOGGER.info(
f"Test results can be found at {os.path.join(target_upload_location, log_file)}"
)
result_statement, throughput = _print_results_of_test(
os.path.join(test_dir, log_file), processor)
throughput /= num_nodes
assert run_out.ok, (
f"Benchmark Test failed with return code {run_out.return_code}. "
f"Test results can be found at {os.path.join(target_upload_location, log_file)}"
)
threshold_table = ((TENSORFLOW_SM_TRAINING_CPU_1NODE_THRESHOLD if num_nodes
== 1 else TENSORFLOW_SM_TRAINING_CPU_4NODE_THRESHOLD)
if processor == "cpu" else
TENSORFLOW_SM_TRAINING_GPU_1NODE_THRESHOLD if num_nodes
== 1 else TENSORFLOW_SM_TRAINING_GPU_4NODE_THRESHOLD)
threshold = get_threshold_for_image(framework_version, threshold_table)
LOGGER.info(
f"tensorflow {framework_version} sagemaker training {device_cuda_str} {py_version} "
f"imagenet {num_nodes} nodes Throughput: {throughput} images/sec, threshold: {threshold} images/sec"
)
assert throughput > threshold, (
f"tensorflow {framework_version} sagemaker training {processor} {py_version} imagenet {num_nodes} nodes "
f"Benchmark Result {throughput} does not reach the threshold {threshold}"
)
class WorkspaceContext:
def __init__(self, root):
self.root = root
self.mrover_build_root = os.path.join(os.path.expanduser('~'),
'.mrover')
self.jarvis_root = os.path.join(root, 'jarvis_files')
self.third_party_root = os.path.join(root, '3rdparty')
self.build_intermediate = os.path.join(self.mrover_build_root,
'scratch')
self.product_env = os.path.join(self.mrover_build_root, 'build_env')
self.jarvis_env = os.path.join(self.mrover_build_root, 'jarvis_env')
self.mbed_env = os.path.join(self.mrover_build_root, 'mbed_env')
self.hash_store = os.path.join(self.mrover_build_root,
'project_hashes')
self.templates = Environment(loader=FileSystemLoader(
os.path.join(self.jarvis_root, 'templates')))
self.ctx = Context()
def ensure_dir(self, d):
"""
Creates a directory if it does not exist. After invocation of this
function, you can be ensured the directory exists.
Parameters:
d - the path to the directory to create.
Raises:
BuildError if there is a file named `d`.
"""
if not os.path.exists(d):
os.makedirs(d)
else:
if not os.path.isdir(d):
raise BuildError("{} already exists and is a file".format(d))
def ensure_build_dirs(self):
"""
Ensures the build directory structure exists.
"""
self.ensure_dir(self.mrover_build_root)
self.ensure_dir(self.hash_store)
def ensure_product_env(self, clear=False):
"""
Ensures the product venv existence. If clear is True, re-creates
the product venv.
"""
self.ensure_build_dirs()
if not os.path.isdir(self.product_env) and not clear:
venv.create(self.product_env,
clear=clear,
symlinks=True,
with_pip=True)
def ensure_mbed_env(self):
"""
Ensures the mbed venv exitence.
"""
self.ensure_build_dirs()
if not os.path.isdir(self.mbed_env):
self.ctx.run('virtualenv --python=python2 {}'.format(
self.mbed_env))
@contextmanager
def inside_product_env(self):
"""
A context manager for activating the product venv.
"""
with self.ctx.prefix("source {}/bin/activate".format(
self.product_env)):
yield
@contextmanager
def inside_mbed_env(self):
"""
A context manager for activating the mbed venv.
"""
with self.ctx.prefix("source {}/bin/activate".format(self.mbed_env)):
yield
def template(self, name, **kwargs):
"""
Templates out a file and returns the rendered copy.
"""
tpl = self.templates.get_template(name)
return tpl.render(**kwargs)
@contextmanager
def cd(self, *args):
with self.ctx.cd(*args):
yield
def run(self, *args, **kwargs):
return self.ctx.run(*args, **kwargs)
def get_product_file(self, *args):
return os.path.join(self.product_env, *args)
def get_jarvis_file(self, *args):
return os.path.join(self.jarvis_env, *args)
def get_mbed_file(self, *args):
return os.path.join(self.mbed_env, *args)
@contextmanager
def intermediate(self, name, cleanup=False):
"""
Create an intermediate build directory, then change directory to it.
"""
intermediate = os.path.join(self.build_intermediate, name)
self.ensure_dir(intermediate)
if os.listdir(intermediate) and cleanup:
shutil.rmtree(intermediate)
self.ensure_dir(intermediate)
with self.cd(intermediate):
yield intermediate
if cleanup:
shutil.rmtree(intermediate)
def run_sm_profiler_tests(image, profiler_tests_dir, test_file, processor):
"""
Testrunner to execute SM profiler tests from DLC repo
"""
ctx = Context()
# Install profiler requirements only once - pytest-rerunfailures has a known issue
# with the latest pytest https://github.com/pytest-dev/pytest-rerunfailures/issues/128
try:
ctx.run(
"pip install -r "
"https://raw.githubusercontent.com/awslabs/sagemaker-debugger/master/config/profiler/requirements.txt && "
"pip install smdebug && "
"pip uninstall -y pytest-rerunfailures",
hide=True,
)
except UnexpectedExit:
# Wait a minute and a half if we get an invoke failure - since smprofiler test requirements can be flaky
time.sleep(90)
framework, version = get_framework_and_version_from_tag(image)
# Conditionally set sm data parallel tests, based on config file rules from link below:
# https://github.com/awslabs/sagemaker-debugger/tree/master/config/profiler
enable_sm_data_parallel_tests = "true"
if framework == "pytorch" and Version(version) < Version("1.6"):
enable_sm_data_parallel_tests = "false"
if framework == "tensorflow" and Version(version) < Version("2.3"):
enable_sm_data_parallel_tests = "false"
# Set SMProfiler specific environment variables
smprof_configs = {
"use_current_branch": "false",
"enable_smdataparallel_tests": enable_sm_data_parallel_tests,
"force_run_tests": "false",
"framework": framework,
"build_type": "release"
}
# Command to set all necessary environment variables
export_cmd = " && ".join(f"export {key}={val}"
for key, val in smprof_configs.items())
export_cmd = f"{export_cmd} && export ENV_CPU_TRAIN_IMAGE=test && export ENV_GPU_TRAIN_IMAGE=test && " \
f"export ENV_{processor.upper()}_TRAIN_IMAGE={image}"
test_results_outfile = os.path.join(
os.getcwd(), f"{get_container_name('smprof', image)}.txt")
with ctx.prefix(f"cd {profiler_tests_dir}"):
with ctx.prefix(f"cd sagemaker-tests && {export_cmd}"):
try:
ctx.run(
f"pytest --json-report --json-report-file={test_results_outfile} -n=auto "
f"-v -s -W=ignore tests/{test_file}::test_{processor}_jobs",
hide=True,
)
with open(test_results_outfile) as outfile:
result_data = json.load(outfile)
LOGGER.info(
f"Tests passed on {image}; Results:\n{json.dumps(result_data, indent=4)}"
)
except Exception as e:
if os.path.exists(test_results_outfile):
with open(test_results_outfile) as outfile:
result_data = json.load(outfile)
raise SMProfilerRCTestFailure(
f"Failed SM Profiler tests. Results:\n{json.dumps(result_data, indent=4)}"
) from e
raise
def test_tensorflow_sagemaker_training_performance(tensorflow_training,
num_nodes, region):
"""
Run TF sagemaker training performance tests
Additonal context: Setup for this function is performed by 'setup_sm_benchmark_tf_train_env' -- this installs
some prerequisite packages, clones some repos, and creates a virtualenv called sm_benchmark_venv.
TODO: Refactor the above setup function to be more obviously connected to this function,
TODO: and install requirements via a requirements.txt file
:param tensorflow_training: ECR image URI
:param num_nodes: Number of nodes to run on
:param region: AWS region
"""
framework_version = re.search(r"[1,2](\.\d+){2}",
tensorflow_training).group()
if framework_version.startswith("1."):
pytest.skip("Skipping benchmark test on TF 1.x images.")
processor = "gpu" if "gpu" in tensorflow_training else "cpu"
ec2_instance_type = "p3.16xlarge" if processor == "gpu" else "c5.18xlarge"
py_version = "py2" if "py2" in tensorflow_training else "py37" if "py37" in tensorflow_training else "py3"
time_str = time.strftime('%Y-%m-%d-%H-%M-%S')
commit_info = os.getenv("CODEBUILD_RESOLVED_SOURCE_VERSION")
target_upload_location = os.path.join(BENCHMARK_RESULTS_S3_BUCKET,
"tensorflow", framework_version,
"sagemaker", "training", processor,
py_version)
training_job_name = (
f"tf{framework_version[0]}-tr-bench-{processor}-{num_nodes}-node-{py_version}"
f"-{commit_info[:7]}-{time_str}")
# Inserting random sleep because this test starts multiple training jobs around the same time, resulting in
# a throttling error for SageMaker APIs.
time.sleep(Random(x=training_job_name).random() * 60)
test_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),
"resources")
venv_dir = os.path.join(test_dir, "sm_benchmark_venv")
ctx = Context()
with ctx.cd(test_dir), ctx.prefix(f"source {venv_dir}/bin/activate"):
log_file = f"results-{commit_info}-{time_str}-{framework_version}-{processor}-{py_version}-{num_nodes}-node.txt"
run_out = ctx.run(
f"timeout 45m python tf_sm_benchmark.py "
f"--framework-version {framework_version} "
f"--image-uri {tensorflow_training} "
f"--instance-type ml.{ec2_instance_type} "
f"--node-count {num_nodes} "
f"--python {py_version} "
f"--region {region} "
f"--job-name {training_job_name}"
f"2>&1 | tee {log_file}",
warn=True,
echo=True)
if not (run_out.ok or run_out.return_code == 124):
target_upload_location = os.path.join(target_upload_location,
"failure_log")
ctx.run(
f"aws s3 cp {os.path.join(test_dir, log_file)} {os.path.join(target_upload_location, log_file)}"
)
LOGGER.info(
f"Test results can be found at {os.path.join(target_upload_location, log_file)}"
)
_print_results_of_test(os.path.join(test_dir, log_file), processor)
assert run_out.ok, (
f"Benchmark Test failed with return code {run_out.return_code}. "
f"Test results can be found at {os.path.join(target_upload_location, log_file)}"
)
def test_mxnet_sagemaker_training_performance(mxnet_training, num_nodes,
region, gpu_only, py3_only):
"""
Run MX sagemaker training performance test
Additional context: Setup for this function is performed by 'setup_sm_benchmark_mx_train_env' -- this installs
some prerequisite packages, pulls required script, and creates a virtualenv called sm_benchmark_venv.
The training script mxnet_imagenet_resnet50.py is invoked via a shell script smtrain-resnet50-imagenet.sh
The shell script sets num-epochs to 40. This parameter is configurable.
TODO: Refactor the above setup function to be more obviously connected to this function,
TODO: and install requirements via a requirements.txt file
TODO: Change latency [time/epoch] metric to Throughput metric
:param mxnet_training: ECR image URI
:param num_nodes: Number of nodes to run on
:param region: AWS region
"""
_, framework_version = get_framework_and_version_from_tag(mxnet_training)
device_cuda_str = f"gpu-{get_cuda_version_from_tag(mxnet_training)}"
py_version = "py37" if "py37" in mxnet_training else "py2" if "py2" in mxnet_training else "py3"
ec2_instance_type = "p3.16xlarge"
time_str = time.strftime('%Y-%m-%d-%H-%M-%S')
commit_info = os.getenv("CODEBUILD_RESOLVED_SOURCE_VERSION", "manual")
target_upload_location = os.path.join(BENCHMARK_RESULTS_S3_BUCKET, "mxnet",
framework_version, "sagemaker",
"training", device_cuda_str,
py_version)
training_job_name = f"mx-tr-bench-{device_cuda_str}-{num_nodes}-node-{py_version}-{commit_info[:7]}-{time_str}"
test_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),
"resources")
venv_dir = os.path.join(test_dir, "sm_benchmark_venv")
ctx = Context()
with ctx.cd(test_dir), ctx.prefix(f"source {venv_dir}/bin/activate"):
log_file = f"results-{commit_info}-{time_str}-{num_nodes}-node.txt"
run_out = ctx.run(
f"timeout 90m python mx_sm_benchmark.py "
f"--framework-version {framework_version} "
f"--image-uri {mxnet_training} "
f"--instance-type ml.{ec2_instance_type} "
f"--node-count {num_nodes} "
f"--python {py_version} "
f"--region {region} "
f"--job-name {training_job_name} "
f"2>&1 | tee {log_file}",
warn=True,
echo=True)
if not run_out.ok:
target_upload_location = os.path.join(target_upload_location,
"failure_log")
ctx.run(
f"aws s3 cp {os.path.join(test_dir, log_file)} {os.path.join(target_upload_location, log_file)}",
warn=True,
echo=True)
LOGGER.info(
f"Test results can be found at {os.path.join(target_upload_location, log_file)}"
)
assert run_out.ok, (
f"Benchmark Test failed with return code {run_out.return_code}. "
f"Test results can be found at {os.path.join(target_upload_location, log_file)}"
)
result_statement, time_val, accuracy = _print_results_of_test(
os.path.join(test_dir, log_file))
accuracy_threshold = get_threshold_for_image(
framework_version, MXNET_TRAINING_GPU_IMAGENET_ACCURACY_THRESHOLD)
assert accuracy > accuracy_threshold, (
f"mxnet {framework_version} sagemaker training {py_version} imagenet {num_nodes} nodes "
f"Benchmark Result {accuracy} does not reach the threshold accuracy {accuracy_threshold}"
)
time_threshold = get_threshold_for_image(
framework_version, MXNET_TRAINING_GPU_IMAGENET_LATENCY_THRESHOLD)
assert time_val < time_threshold, (
f"mxnet {framework_version} sagemaker training {py_version} imagenet {num_nodes} nodes "
f"Benchmark Result {time_val} does not reach the threshold latency {time_threshold}"
)
def run_sm_perf_test(image_uri, xla, num_nodes, region, threshold=None):
"""
Run TF sagemaker training performance tests
Additional context: Setup for this function is performed by 'setup_sm_benchmark_tf_train_env' -- this installs
some prerequisite packages, clones some repos, and creates a virtualenv called sm_benchmark_venv.
TODO: Refactor the above setup function to be more obviously connected to this function,
TODO: and install requirements via a requirements.txt file
:param image_uri: ECR image URI
:param xla: [ True | False ] Enable XLA acceleration
:param num_nodes: Number of nodes to run on
:param region: AWS region
This function was inspired by deep-learning-containers/test/dlc_tests/benchmark/sagemaker/tensorflow/training/test_performance_tensorflow_sm_training.py
"""
_, framework_version = get_framework_and_version_from_tag(image_uri)
processor = "xla" if xla else "gpu"
device_cuda_str = f"{processor}-{get_cuda_version_from_tag(image_uri)}"
'''
TODO: Switch to p3.16xlarge when EC2 availability issues are resolved
'''
ec2_instance_type = "p3.8xlarge"
py_version = "py2" if "py2" in image_uri else "py37" if "py37" in image_uri else "py3"
time_str = time.strftime("%Y-%m-%d-%H-%M-%S")
commit_info = os.getenv("CODEBUILD_RESOLVED_SOURCE_VERSION")
target_upload_location = os.path.join(
BENCHMARK_RESULTS_S3_BUCKET, "xla", "tensorflow", framework_version, "sagemaker", "training", device_cuda_str, py_version
)
training_job_name = (
f"opt-tf{framework_version[0]}-bench-{device_cuda_str}-{num_nodes}-node-{py_version}-{commit_info[:7]}-{time_str}"
)
# Inserting random sleep because this test starts multiple training jobs around the same time, resulting in
# a throttling error for SageMaker APIs.
time.sleep(Random(x=training_job_name).random() * 60)
test_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "resources")
venv_dir = os.path.join(test_dir, "sm_benchmark_venv")
ctx = Context()
with ctx.cd(test_dir), ctx.prefix(f"source {venv_dir}/bin/activate"):
log_file = (
f"results-{commit_info}-{time_str}-optimized-tf{framework_version}-{device_cuda_str}-{py_version}-{num_nodes}-node.txt"
)
run_out = ctx.run(
f"timeout 45m python tf_sm_benchmark.py "
f"--framework-version {framework_version} "
f"--image-uri {image_uri} "
f"--instance-type ml.{ec2_instance_type} "
f"--node-count {num_nodes} "
f"--python {py_version} "
f"--region {region} "
f"--job-name {training_job_name} "
f"--xla-{'on' if xla else 'off'} "
f"2>&1 | tee {log_file}",
warn=True,
echo=True,
)
if not (run_out.ok or run_out.return_code == 124):
target_upload_location = os.path.join(target_upload_location, "failure_log")
ctx.run(f"aws s3 cp {os.path.join(test_dir, log_file)} {os.path.join(target_upload_location, log_file)}")
LOGGER.info(f"Test results can be found at {os.path.join(target_upload_location, log_file)}")
result_statement, throughput = _print_results_of_test(os.path.join(test_dir, log_file))
throughput /= num_nodes
assert run_out.ok, (
f"Benchmark Test failed with return code {run_out.return_code}. "
f"Test results can be found at {os.path.join(target_upload_location, log_file)}"
)
LOGGER.info(
f"optimized-tensorflow-{framework_version} sagemaker training {ec2_instance_type} {device_cuda_str} {py_version} "
f"imagenet {num_nodes} nodes Throughput: {throughput} images/sec, threshold: {threshold} images/sec"
)
if threshold:
assert throughput > threshold, (
f"optimized-tensorflow-{framework_version} sagemaker training {ec2_instance_type} {device_cuda_str} {py_version} imagenet {num_nodes} nodes "
f"Regression Benchmark Result {throughput} does not reach the threshold {threshold}"
)
return throughput
def test_tensorflow_sagemaker_training_performance(tensorflow_training,
num_nodes, region):
framework_version = re.search(r"[1,2](\.\d+){2}",
tensorflow_training).group()
if framework_version.startswith("1."):
pytest.skip("Skipping benchmark test on TF 1.x images.")
processor = "gpu" if "gpu" in tensorflow_training else "cpu"
ec2_instance_type = "p3.16xlarge" if processor == "gpu" else "c5.18xlarge"
py_version = "py2" if "py2" in tensorflow_training else "py37" if "py37" in tensorflow_training else "py3"
time_str = time.strftime('%Y-%m-%d-%H-%M-%S')
commit_info = os.getenv("CODEBUILD_RESOLVED_SOURCE_VERSION")
target_upload_location = os.path.join(BENCHMARK_RESULTS_S3_BUCKET,
"tensorflow", framework_version,
"sagemaker", "training", processor,
py_version)
training_job_name = (
f"tf{framework_version[0]}-tr-bench-{processor}-{num_nodes}-node-{py_version}"
f"-{commit_info[:7]}-{time_str}")
# Inserting random sleep because this test starts multiple training jobs around the same time, resulting in
# a throttling error for SageMaker APIs.
time.sleep(Random(x=training_job_name).random() * 60)
test_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),
"resources")
venv_dir = os.path.join(test_dir, "sm_benchmark_venv")
ctx = Context()
with ctx.cd(test_dir), ctx.prefix(f"source {venv_dir}/bin/activate"):
log_file = f"results-{commit_info}-{time_str}-{num_nodes}-node.txt"
run_out = ctx.run(
f"timeout 45m python tf_sm_benchmark.py "
f"--framework-version {framework_version} "
f"--image-uri {tensorflow_training} "
f"--instance-type ml.{ec2_instance_type} "
f"--node-count {num_nodes} "
f"--python {py_version} "
f"--region {region} "
f"--job-name {training_job_name}"
f"2>&1 > {log_file}",
warn=True,
echo=True)
if not (run_out.ok or run_out.return_code == 124):
target_upload_location = os.path.join(target_upload_location,
"failure_log")
ctx.run(
f"aws s3 cp {os.path.join(test_dir, log_file)} {os.path.join(target_upload_location, log_file)}"
)
LOGGER.info(
f"Test results can be found at {os.path.join(target_upload_location, log_file)}"
)
assert run_out.ok, (
f"Benchmark Test failed with return code {run_out.return_code}. "
f"Test results can be found at {os.path.join(target_upload_location, log_file)}"
)
