def run_ec2_tensorflow_inference(image_uri, ec2_connection, grpc_port, region):
repo_name, image_tag = image_uri.split("/")[-1].split(":")
container_name = f"{repo_name}-{image_tag}-ec2"
framework_version = get_tensorflow_framework_version(image_uri)
home_dir = ec2_connection.run("echo $HOME").stdout.strip('\n')
serving_folder_path = os.path.join(home_dir, "serving")
model_path = os.path.join(serving_folder_path, "models", "mnist")
mnist_client_path = os.path.join(serving_folder_path, "tensorflow_serving",
"example", "mnist_client.py")
docker_cmd = "nvidia-docker" if "gpu" in image_uri else "docker"
docker_run_cmd = (
f"{docker_cmd} run -id --name {container_name} -p {grpc_port}:8500 "
f"--mount type=bind,source={model_path},target=/models/mnist -e MODEL_NAME=mnist"
f" {image_uri}")
try:
host_setup_for_tensorflow_inference(serving_folder_path,
framework_version, ec2_connection)
sleep(2)
train_mnist_model(serving_folder_path, ec2_connection)
sleep(10)
ec2_connection.run(
f"$(aws ecr get-login --no-include-email --region {region})",
hide=True)
LOGGER.info(docker_run_cmd)
ec2_connection.run(docker_run_cmd, hide=True)
sleep(20)
test_utils.request_tensorflow_inference_grpc(
script_file_path=mnist_client_path,
port=grpc_port,
connection=ec2_connection)
finally:
ec2_connection.run(f"docker rm -f {container_name}",
warn=True,
hide=True)
def run_ec2_tensorflow_inference(image_uri,
ec2_connection,
ec2_instance_ami,
grpc_port,
region,
telemetry_mode=False):
repo_name, image_tag = image_uri.split("/")[-1].split(":")
container_name = f"{repo_name}-{image_tag}-ec2"
framework_version = get_tensorflow_framework_version(image_uri)
home_dir = ec2_connection.run("echo $HOME").stdout.strip('\n')
serving_folder_path = os.path.join(home_dir, "serving")
model_path = os.path.join(serving_folder_path, "models", "mnist")
python_invoker = test_utils.get_python_invoker(ec2_instance_ami)
mnist_client_path = os.path.join(serving_folder_path, "tensorflow_serving",
"example", "mnist_client.py")
is_neuron = "neuron" in image_uri
docker_cmd = "nvidia-docker" if "gpu" in image_uri else "docker"
if is_neuron:
docker_run_cmd = (
f"{docker_cmd} run -id --name {container_name} -p {grpc_port}:8500 "
f"--device=/dev/neuron0 --net=host --cap-add IPC_LOCK "
f"-e NEURON_MONITOR_CW_REGION=us-east-1 -e NEURON_MONITOR_CW_NAMESPACE=tf1 "
f"--mount type=bind,source={model_path},target=/models/mnist -e TEST_MODE=1 -e MODEL_NAME=mnist"
f" {image_uri}")
else:
docker_run_cmd = (
f"{docker_cmd} run -id --name {container_name} -p {grpc_port}:8500 "
f"--mount type=bind,source={model_path},target=/models/mnist -e TEST_MODE=1 -e MODEL_NAME=mnist"
f" {image_uri}")
try:
host_setup_for_tensorflow_inference(serving_folder_path,
framework_version, ec2_connection,
is_neuron, 'mnist', python_invoker)
sleep(2)
if not is_neuron:
train_mnist_model(serving_folder_path, ec2_connection,
python_invoker)
sleep(10)
ec2_connection.run(
f"$(aws ecr get-login --no-include-email --region {region})",
hide=True)
LOGGER.info(docker_run_cmd)
ec2_connection.run(docker_run_cmd, hide=True)
sleep(20)
test_utils.request_tensorflow_inference_grpc(
script_file_path=mnist_client_path,
port=grpc_port,
connection=ec2_connection,
ec2_instance_ami=ec2_instance_ami)
if telemetry_mode:
check_telemetry(ec2_connection, container_name)
finally:
ec2_connection.run(f"docker rm -f {container_name}",
warn=True,
hide=True)
