def run_ec2_tensorflow_inference(image_uri, ec2_connection, grpc_port, region): repo_name, image_tag = image_uri.split("/")[-1].split(":") container_name = f"{repo_name}-{image_tag}-ec2" framework_version = get_tensorflow_framework_version(image_uri) home_dir = ec2_connection.run("echo $HOME").stdout.strip('\n') serving_folder_path = os.path.join(home_dir, "serving") model_path = os.path.join(serving_folder_path, "models", "mnist") mnist_client_path = os.path.join(serving_folder_path, "tensorflow_serving", "example", "mnist_client.py") docker_cmd = "nvidia-docker" if "gpu" in image_uri else "docker" docker_run_cmd = ( f"{docker_cmd} run -id --name {container_name} -p {grpc_port}:8500 " f"--mount type=bind,source={model_path},target=/models/mnist -e MODEL_NAME=mnist" f" {image_uri}") try: host_setup_for_tensorflow_inference(serving_folder_path, framework_version, ec2_connection) sleep(2) train_mnist_model(serving_folder_path, ec2_connection) sleep(10) ec2_connection.run( f"$(aws ecr get-login --no-include-email --region {region})", hide=True) LOGGER.info(docker_run_cmd) ec2_connection.run(docker_run_cmd, hide=True) sleep(20) test_utils.request_tensorflow_inference_grpc( script_file_path=mnist_client_path, port=grpc_port, connection=ec2_connection) finally: ec2_connection.run(f"docker rm -f {container_name}", warn=True, hide=True)
def run_ec2_tensorflow_inference(image_uri, ec2_connection, ec2_instance_ami, grpc_port, region, telemetry_mode=False): repo_name, image_tag = image_uri.split("/")[-1].split(":") container_name = f"{repo_name}-{image_tag}-ec2" framework_version = get_tensorflow_framework_version(image_uri) home_dir = ec2_connection.run("echo $HOME").stdout.strip('\n') serving_folder_path = os.path.join(home_dir, "serving") model_path = os.path.join(serving_folder_path, "models", "mnist") python_invoker = test_utils.get_python_invoker(ec2_instance_ami) mnist_client_path = os.path.join(serving_folder_path, "tensorflow_serving", "example", "mnist_client.py") is_neuron = "neuron" in image_uri docker_cmd = "nvidia-docker" if "gpu" in image_uri else "docker" if is_neuron: docker_run_cmd = ( f"{docker_cmd} run -id --name {container_name} -p {grpc_port}:8500 " f"--device=/dev/neuron0 --net=host --cap-add IPC_LOCK " f"-e NEURON_MONITOR_CW_REGION=us-east-1 -e NEURON_MONITOR_CW_NAMESPACE=tf1 " f"--mount type=bind,source={model_path},target=/models/mnist -e TEST_MODE=1 -e MODEL_NAME=mnist" f" {image_uri}") else: docker_run_cmd = ( f"{docker_cmd} run -id --name {container_name} -p {grpc_port}:8500 " f"--mount type=bind,source={model_path},target=/models/mnist -e TEST_MODE=1 -e MODEL_NAME=mnist" f" {image_uri}") try: host_setup_for_tensorflow_inference(serving_folder_path, framework_version, ec2_connection, is_neuron, 'mnist', python_invoker) sleep(2) if not is_neuron: train_mnist_model(serving_folder_path, ec2_connection, python_invoker) sleep(10) ec2_connection.run( f"$(aws ecr get-login --no-include-email --region {region})", hide=True) LOGGER.info(docker_run_cmd) ec2_connection.run(docker_run_cmd, hide=True) sleep(20) test_utils.request_tensorflow_inference_grpc( script_file_path=mnist_client_path, port=grpc_port, connection=ec2_connection, ec2_instance_ami=ec2_instance_ami) if telemetry_mode: check_telemetry(ec2_connection, container_name) finally: ec2_connection.run(f"docker rm -f {container_name}", warn=True, hide=True)