def start_server_update_flow_latest(request, get_image, get_test_dir,
get_docker_context):
client = get_docker_context
path_to_mount = get_test_dir + '/saved_models'
update_test_dir = path_to_mount + '/update-{}/'.format(get_tests_suffix())
# ensure model dir is empty before starting OVMS
shutil.rmtree(update_test_dir, ignore_errors=True)
volumes_dict = {'{}'.format(path_to_mount): {'bind': '/opt/ml',
'mode': 'ro'}}
grpc_port, rest_port = get_ports_for_fixture(port_suffix="03")
command = "/ie-serving-py/start_server.sh ie_serving model " \
"--model_name resnet --model_path /opt/ml/update-{} " \
"--port {} --rest_port {} --grpc_workers 1 --nireq 1".\
format(get_tests_suffix(), grpc_port, rest_port)
container = client.containers.run(image=get_image, detach=True,
name='ie-serving-py-test-update-'
'latest-{}'.
format(get_tests_suffix()),
ports={'{}/tcp'.format(grpc_port):
grpc_port,
'{}/tcp'.format(rest_port):
rest_port},
remove=True, volumes=volumes_dict,
command=command)
request.addfinalizer(container.kill)
running = wait_endpoint_setup(container)
assert running is True, "docker container was not started successfully"
return container, {"grpc_port": grpc_port, "rest_port": rest_port}
def start_server_single_model_from_minio(request, get_docker_network,
get_minio_server_s3, get_image,
get_test_dir, get_docker_context):
network = get_docker_network
AWS_ACCESS_KEY_ID = os.getenv('MINIO_ACCESS_KEY')
AWS_SECRET_ACCESS_KEY = os.getenv('MINIO_SECRET_KEY')
AWS_REGION = os.getenv('AWS_REGION')
_, ports = get_minio_server_s3
grpc_port = ports["grpc_port"]
minio_endpoint = 'http://minio.locals3-{}.com:{}'.format(
get_tests_suffix(), grpc_port)
envs = [
'MINIO_ACCESS_KEY=' + AWS_ACCESS_KEY_ID,
'MINIO_SECRET_KEY=' + AWS_SECRET_ACCESS_KEY,
'AWS_ACCESS_KEY_ID=' + AWS_ACCESS_KEY_ID,
'AWS_SECRET_ACCESS_KEY=' + AWS_SECRET_ACCESS_KEY,
'AWS_REGION=' + AWS_REGION, 'S3_ENDPOINT=' + minio_endpoint
]
client = get_docker_context
grpc_port, rest_port = get_ports_for_fixture(port_suffix="11")
command = "/ie-serving-py/start_server.sh ie_serving model " \
"--model_name resnet " \
"--model_path s3://inference/resnet_v1_50 " \
"--port {}".format(grpc_port)
container = client.containers.run(
image=get_image,
detach=True,
name='ie-serving-test-'
'single-minio-{}'.format(get_tests_suffix()),
ports={'{}/tcp'.format(grpc_port): grpc_port},
remove=True,
environment=envs,
command=command,
network=network.name)
request.addfinalizer(container.kill)
running = wait_endpoint_setup(container)
assert running is True, "docker container was not started successfully"
return container, {"grpc_port": grpc_port, "rest_port": rest_port}
def start_server_single_model_from_gc(request, get_image, get_test_dir,
get_docker_context):
client = get_docker_context
grpc_port, rest_port = get_ports_for_fixture(port_suffix="08")
command = "/ie-serving-py/start_server.sh ie_serving model " \
"--model_name resnet " \
"--model_path " \
"gs://public-artifacts/intelai_public_models/resnet_50_i8/ " \
"--port " + str(grpc_port) + " --target_device CPU --nireq 4" \
" --plugin_config " \
"\"{\\\"CPU_THROUGHPUT_STREAMS\\\": \\\"2\\\", " \
"\\\"CPU_THREADS_NUM\\\": \\\"4\\\"}\""
envs = ['https_proxy=' + os.getenv('https_proxy', "")]
container = client.containers.run(
image=get_image,
detach=True,
name='ie-serving-py-test-single-gs-{}'.format(get_tests_suffix()),
ports={'{}/tcp'.format(grpc_port): grpc_port},
remove=True,
environment=envs,
command=command)
request.addfinalizer(container.kill)
running = wait_endpoint_setup(container)
assert running is True, "docker container was not started successfully"
return container, {"grpc_port": grpc_port, "rest_port": rest_port}
def start_server_single_model_from_s3(request, get_image, get_test_dir,
get_docker_context):
AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID')
AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY')
AWS_REGION = os.getenv('AWS_REGION')
client = get_docker_context
envs = [
'AWS_ACCESS_KEY_ID=' + AWS_ACCESS_KEY_ID,
'AWS_SECRET_ACCESS_KEY=' + AWS_SECRET_ACCESS_KEY,
'AWS_REGION=' + AWS_REGION
]
grpc_port, rest_port = get_ports_for_fixture(port_suffix="09")
command = "/ie-serving-py/start_server.sh ie_serving model " \
"--model_name resnet " \
"--model_path s3://inference-test-aipg/resnet_v1_50 " \
"--port {}".format(grpc_port)
container = client.containers.run(
image=get_image,
detach=True,
name='ie-serving-py-test-single-s3-{}'.format(get_tests_suffix()),
ports={'{}/tcp'.format(grpc_port): grpc_port},
remove=True,
environment=envs,
command=command)
request.addfinalizer(container.kill)
running = wait_endpoint_setup(container)
assert running is True, "docker container was not started successfully"
return container, {"grpc_port": grpc_port, "rest_port": rest_port}
def start_server_model_ver_policy(request, get_image, get_test_dir,
get_docker_context):
shutil.copyfile('tests/functional/model_version_policy_config.json',
get_test_dir +
'/saved_models/model_ver_policy_config.json')
shutil.copyfile('tests/functional/mapping_config.json',
get_test_dir + '/saved_models/model_ver/3/'
'mapping_config.json')
client = get_docker_context
volumes_dict = {'{}'.format(get_test_dir + '/saved_models/'):
{'bind': '/opt/ml', 'mode': 'ro'}}
grpc_port, rest_port = get_ports_for_fixture(port_suffix="18")
command = "/ie-serving-py/start_server.sh ie_serving config " \
"--config_path /opt/ml/model_ver_policy_config.json " \
"--port {} --rest_port {}".format(grpc_port, rest_port)
container = client.containers.run(image=get_image, detach=True,
name='ie-serving-py-test-policy-{}'.
format(get_tests_suffix()),
ports={'{}/tcp'.format(grpc_port):
grpc_port,
'{}/tcp'.format(rest_port):
rest_port},
remove=True, volumes=volumes_dict,
command=command)
request.addfinalizer(container.kill)
running = wait_endpoint_setup(container)
assert running is True, "docker container was not started successfully"
return container, {"grpc_port": grpc_port, "rest_port": rest_port}
def clean_hanging_docker_resources():
client = get_docker_client()
containers = client.containers.list(all=True)
networks = client.networks.list()
tests_suffix = get_tests_suffix()
clean_hanging_containers(containers, tests_suffix)
clean_hanging_networks(networks, tests_suffix)
def start_server_batch_model_auto(request, get_image, get_test_dir,
get_docker_context):
client = get_docker_context
path_to_mount = get_test_dir + '/saved_models/'
volumes_dict = {
'{}'.format(path_to_mount): {
'bind': '/opt/ml',
'mode': 'ro'
}
}
grpc_port, rest_port = get_ports_for_fixture(port_suffix="14")
command = "/ie-serving-py/start_server.sh ie_serving model " \
"--model_name resnet --model_path /opt/ml/resnet_V1_50_batch8 " \
"--port {} --batch_size auto --rest_port {}".\
format(grpc_port, rest_port)
container = client.containers.run(
image=get_image,
detach=True,
name='ie-serving-py-test-autobatch-{}'.format(get_tests_suffix()),
ports={
'{}/tcp'.format(grpc_port): grpc_port,
'{}/tcp'.format(rest_port): rest_port
},
remove=True,
volumes=volumes_dict,
command=command)
request.addfinalizer(container.kill)
running = wait_endpoint_setup(container)
assert running is True, "docker container was not started successfully"
return container, {"grpc_port": grpc_port, "rest_port": rest_port}
def get_docker_network(request, get_docker_context):
client = get_docker_context
existing = None
try:
existing = client.networks.get("minio-network-{}".format(
get_tests_suffix()))
except Exception as e:
pass
if existing is not None:
existing.remove()
network = client.networks.create("minio-network-{}".format(
get_tests_suffix()))
request.addfinalizer(network.remove)
return network
def start_server_update_flow_specific(request, get_image, get_test_dir,
get_docker_context):
client = get_docker_context
path_to_mount = get_test_dir + '/saved_models'
update_test_dir = path_to_mount + '/update-{}/'.format(get_tests_suffix())
# ensure model dir is empty before starting OVMS
shutil.rmtree(update_test_dir, ignore_errors=True)
volumes_dict = {'{}'.format(path_to_mount): {'bind': '/opt/ml',
'mode': 'ro'}}
grpc_port, rest_port = get_ports_for_fixture(port_suffix="04")
command = '/ie-serving-py/start_server.sh ie_serving model ' \
'--model_name resnet --model_path ' \
'/opt/ml/update-' + get_tests_suffix() + \
' --port ' + str(grpc_port) + ' --model_version_policy' \
' \'{"specific": { "versions":[1, 3, 4] }}\' ' \
'--rest_port ' + str(rest_port)
container = client.containers.run(image=get_image, detach=True,
name='ie-serving-py-test-'
'update-specific-{}'.
format(get_tests_suffix()),
ports={'{}/tcp'.format(grpc_port):
grpc_port,
'{}/tcp'.format(rest_port):
rest_port},
remove=True, volumes=volumes_dict,
command=command)
request.addfinalizer(container.kill)
running = wait_endpoint_setup(container)
assert running is True, "docker container was not started successfully"
return container, {"grpc_port": grpc_port, "rest_port": rest_port}
def start_server_with_mapping(request, get_image, get_test_dir,
get_docker_context):
shutil.copyfile(
'tests/functional/mapping_config.json', get_test_dir + '/saved_models/'
'age-gender-recognition-retail-0013/1/'
'mapping_config.json')
client = get_docker_context
path_to_mount = get_test_dir + '/saved_models/'
volumes_dict = {
'{}'.format(path_to_mount): {
'bind': '/opt/ml',
'mode': 'ro'
}
}
grpc_port, rest_port = get_ports_for_fixture(port_suffix="06")
command = "/ie-serving-py/start_server.sh ie_serving model " \
"--model_name age_gender " \
"--model_path /opt/ml/age-gender-recognition-retail-0013 " \
"--port {} --rest_port {}".format(grpc_port, rest_port)
container = client.containers.run(
image=get_image,
detach=True,
name='ie-serving-py-test-2-out-{}'.format(get_tests_suffix()),
ports={
'{}/tcp'.format(grpc_port): grpc_port,
'{}/tcp'.format(rest_port): rest_port
},
remove=True,
volumes=volumes_dict,
command=command)
def delete_mapping_file():
path = get_test_dir + '/saved_models/' \
'age-gender-recognition-retail-0013/1/' \
'mapping_config.json'
if os.path.exists(path):
os.remove(path)
request.addfinalizer(delete_mapping_file)
request.addfinalizer(container.kill)
running = wait_endpoint_setup(container)
assert running is True, "docker container was not started successfully"
return container, {"grpc_port": grpc_port, "rest_port": rest_port}
def start_server_single_vehicle_attrib_model(
request, get_image, get_test_dir, vehicle_attributes_model_downloader,
get_docker_context):
print("Downloaded model files:", vehicle_attributes_model_downloader)
client = get_docker_context
path_to_mount = get_test_dir + '/saved_models/'
volumes_dict = {
'{}'.format(path_to_mount): {
'bind': '/opt/ml',
'mode': 'ro'
}
}
grpc_port, rest_port = get_ports_for_fixture(port_suffix="05")
command = "/ie-serving-py/start_server.sh ie_serving model " \
"--model_name vehicle-attributes " \
"--model_path /opt/ml/vehicle-attributes-recognition-barrier-0039 " \
"--port " + str(grpc_port) + " --rest_port " + str(rest_port) + \
" --plugin_config " \
"\"{\\\"CPU_THROUGHPUT_STREAMS\\\": " \
"\\\"CPU_THROUGHPUT_AUTO\\\"}\""
container = \
client.containers.run(
image=get_image,
detach=True,
name='ie-serving-py-test-single-{}'.format(get_tests_suffix()),
ports={'{}/tcp'.format(grpc_port): grpc_port,
'{}/tcp'.format(rest_port): rest_port},
remove=True,
volumes=volumes_dict,
# In this case, slower,
# non-default serialization method is used
environment=[
'SERIALIZATON=_prepare_output_as_AppendArrayToTensorProto'],
command=command)
request.addfinalizer(container.kill)
running = wait_endpoint_setup(container)
assert running is True, "docker container was not started successfully"
return container, {"grpc_port": grpc_port, "rest_port": rest_port}
def convert_model(client, model, output_dir, model_name, input_shape):
files = (os.path.join(output_dir, model_name) + '.bin',
os.path.join(output_dir, model_name) + '.xml')
if os.path.exists(files[0]) and os.path.exists(files[1]):
return files
Path(output_dir).mkdir(parents=True, exist_ok=True)
input_shape_str = '[{}]'.format(','.join(str(i) for i in input_shape))
print("Converting {} to IR with input shape {}...".format(
model, input_shape_str))
input_dir = os.path.dirname(model)
image = 'openvino/ubuntu18_dev:latest'
volumes = {
input_dir: {
'bind': '/mnt/input_dir',
'mode': 'ro'
},
output_dir: {
'bind': '/mnt/output_dir',
'mode': 'rw'
}
}
user_id = os.getuid()
command = ' '.join([
'python3 deployment_tools/model_optimizer/mo.py',
'--input_model /mnt/input_dir/' + os.path.basename(model),
'--model_name ' + model_name, '--output_dir /mnt/output_dir/',
'--input_shape ' + input_shape_str
])
client.containers.run(image=image,
name='convert-model-{}'.format(get_tests_suffix()),
volumes=volumes,
user=user_id,
command=command,
remove=True)
return files
def start_minio_server(request, get_test_dir, get_docker_network,
get_docker_context):
"""sudo docker run -d -p 9099:9000 minio/minio server /data"""
client = get_docker_context
grpc_port, rest_port = get_ports_for_fixture(port_suffix="10")
command = 'server --address ":{}" /data'.format(grpc_port)
client.images.pull('minio/minio:latest')
network = get_docker_network
MINIO_ACCESS_KEY = os.getenv('MINIO_ACCESS_KEY')
MINIO_SECRET_KEY = os.getenv('MINIO_SECRET_KEY')
if MINIO_ACCESS_KEY is None or MINIO_SECRET_KEY is None:
MINIO_ACCESS_KEY = "MINIO_A_KEY"
MINIO_SECRET_KEY = "MINIO_S_KEY"
os.environ["MINIO_ACCESS_KEY"] = "MINIO_A_KEY"
os.environ["MINIO_SECRET_KEY"] = "MINIO_S_KEY"
envs = [
'MINIO_ACCESS_KEY=' + MINIO_ACCESS_KEY,
'MINIO_SECRET_KEY=' + MINIO_SECRET_KEY
]
container = client.containers.run(
image='minio/minio:latest',
detach=True,
name='minio.locals3-{}.com'.format(get_tests_suffix()),
ports={'{}/tcp'.format(grpc_port): grpc_port},
remove=True,
environment=envs,
command=command,
network=network.name)
request.addfinalizer(container.kill)
running = wait_endpoint_setup(container, minio_condition, 30)
assert running is True, "minio container was not started successfully"
return container, {"grpc_port": grpc_port, "rest_port": rest_port}
def start_server_face_detection_model_named_shape(request, get_image,
get_test_dir,
get_docker_context):
client = get_docker_context
path_to_mount = get_test_dir + '/saved_models/'
volumes_dict = {
'{}'.format(path_to_mount): {
'bind': '/opt/ml',
'mode': 'ro'
}
}
grpc_port, rest_port = get_ports_for_fixture(port_suffix="01")
command = "/ie-serving-py/start_server.sh ie_serving model " \
"--model_name face_detection --model_path " \
"/opt/ml/face-detection-retail-0004 " \
"--port " + str(grpc_port) + " --rest_port " + str(rest_port) + \
" --shape \"{\\\"data\\\": \\\"(1, 3, 600, 600)\\\"}\" " \
"--grpc_workers 4 --rest_workers 2 " \
"--nireq 2"
container = client.containers.run(
image=get_image,
detach=True,
name='ie-serving-py-test-named-shape-{}'.format(get_tests_suffix()),
ports={
'{}/tcp'.format(grpc_port): grpc_port,
'{}/tcp'.format(rest_port): rest_port
},
remove=True,
volumes=volumes_dict,
command=command)
request.addfinalizer(container.kill)
running = wait_endpoint_setup(container)
assert running is True, "docker container was not started successfully"
return container, {"grpc_port": grpc_port, "rest_port": rest_port}
def test_specific_version_rest(self, resnet_multiple_batch_sizes,
get_test_dir,
start_server_update_flow_specific):
_, ports = start_server_update_flow_specific
resnet, resnet_bs4, resnet_bs8 = resnet_multiple_batch_sizes
dir = get_test_dir + '/saved_models/' + 'update-{}/'.format(
get_tests_suffix())
# ensure model dir is empty at the beginning
shutil.rmtree(dir, ignore_errors=True)
resnet_copy_dir = copy_model(resnet, 1, dir)
resnet_bs4_copy_dir = copy_model(resnet_bs4, 4, dir)
time.sleep(8)
in_name = 'map/TensorArrayStack/TensorArrayGatherV3'
out_name = 'softmax_tensor'
# Available versions: 1, 4
print("Getting info about resnet model")
model_name = 'resnet'
expected_input_metadata_v1 = {
in_name: {
'dtype': 1,
'shape': [1, 3, 224, 224]
}
}
expected_output_metadata_v1 = {
out_name: {
'dtype': 1,
'shape': [1, 1001]
}
}
rest_url_latest = 'http://localhost:{}/v1/models/resnet/' \
'versions/1/metadata'.format(ports["rest_port"])
response = get_model_metadata_response_rest(rest_url_latest)
input_metadata, output_metadata = model_metadata_response(
response=response)
print(output_metadata)
assert model_name == response.model_spec.name
assert expected_input_metadata_v1 == input_metadata
assert expected_output_metadata_v1 == output_metadata
rest_url = 'http://localhost:{}/v1/models/resnet/metadata'.format(
ports["rest_port"])
response_latest = get_model_metadata_response_rest(rest_url)
print("response", response_latest)
input_metadata_latest, output_metadata_latest = \
model_metadata_response(response=response_latest)
rest_url_v4 = 'http://localhost:{}/v1/models/resnet/' \
'versions/4/metadata'.format(ports["rest_port"])
response_v4 = get_model_metadata_response_rest(rest_url_v4)
print("response", response_v4)
input_metadata_v4, output_metadata_v4 = model_metadata_response(
response=response_latest)
assert response_v4.model_spec.name == response_latest.model_spec.name
assert input_metadata_v4 == input_metadata_latest
assert output_metadata_v4 == output_metadata_latest
# Model status check
rest_status_url = 'http://localhost:{}/v1/models/resnet'.format(
ports["rest_port"])
status_response = get_model_status_response_rest(rest_status_url)
versions_statuses = status_response.model_version_status
assert len(versions_statuses) == 2
for version_status in versions_statuses:
assert version_status.version in [1, 4]
assert version_status.state == ModelVersionState.AVAILABLE
assert version_status.status.error_code == ErrorCode.OK
assert version_status.status.error_message == _ERROR_MESSAGE[
ModelVersionState.AVAILABLE][ErrorCode.OK]
###
shutil.rmtree(resnet_bs4_copy_dir)
resnet_bs8_copy_dir = copy_model(resnet_bs8, 3, dir)
time.sleep(10)
# Available versions: 1, 3
rest_url = 'http://localhost:{}/v1/models/resnet/metadata'.format(
ports["rest_port"])
response_latest = get_model_metadata_response_rest(rest_url)
print("response", response_latest)
input_metadata_latest, output_metadata_latest = \
model_metadata_response(response=response_latest)
rest_url_v3 = 'http://localhost:{}/v1/models/resnet/' \
'versions/3/metadata'.format(ports["rest_port"])
response_v3 = get_model_metadata_response_rest(rest_url_v3)
print("response", response_v3)
input_metadata_v3, output_metadata_v3 = model_metadata_response(
response=response_v3)
assert response_v3.model_spec.name == response_latest.model_spec.name
assert input_metadata_v3 == input_metadata_latest
assert output_metadata_v3 == output_metadata_latest
# Model status check
rest_status_url = 'http://localhost:{}/v1/models/resnet'.format(
ports["rest_port"])
status_response = get_model_status_response_rest(rest_status_url)
versions_statuses = status_response.model_version_status
assert len(versions_statuses) == 3
for version_status in versions_statuses:
assert version_status.version in [1, 3, 4]
if version_status.version == 4:
assert version_status.state == ModelVersionState.END
assert version_status.status.error_code == ErrorCode.OK
assert version_status.status.error_message == _ERROR_MESSAGE[
ModelVersionState.END][ErrorCode.OK]
elif version_status.version == 1 or version_status.version == 3:
assert version_status.state == ModelVersionState.AVAILABLE
assert version_status.status.error_code == ErrorCode.OK
assert version_status.status.error_message == _ERROR_MESSAGE[
ModelVersionState.AVAILABLE][ErrorCode.OK]
###
# Available versions: 1, 3, 4
resnet_bs4_copy_dir = copy_model(resnet_bs4, 4, dir)
time.sleep(10)
rest_url_v1 = 'http://localhost:{}/v1/models/resnet/' \
'versions/1/metadata'.format(ports["rest_port"])
response_v1 = get_model_metadata_response_rest(rest_url_v1)
input_metadata_v1, output_metadata_v1 = model_metadata_response(
response=response_v1)
assert model_name == response_v1.model_spec.name
assert expected_input_metadata_v1 == input_metadata_v1
assert expected_output_metadata_v1 == output_metadata_v1
expected_input_metadata_v3 = {
in_name: {
'dtype': 1,
'shape': [8, 3, 224, 224]
}
}
expected_output_metadata_v3 = {
out_name: {
'dtype': 1,
'shape': [8, 1001]
}
}
rest_url_v3 = 'http://localhost:{}/v1/models/resnet/' \
'versions/3/metadata'.format(ports["rest_port"])
response_v3 = get_model_metadata_response_rest(rest_url_v3)
input_metadata_v3, output_metadata_v3 = model_metadata_response(
response=response_v3)
assert model_name == response_v3.model_spec.name
assert expected_input_metadata_v3 == input_metadata_v3
assert expected_output_metadata_v3 == output_metadata_v3
expected_input_metadata_v4 = {
in_name: {
'dtype': 1,
'shape': [4, 3, 224, 224]
}
}
expected_output_metadata_v4 = {
out_name: {
'dtype': 1,
'shape': [4, 1001]
}
}
response_v4 = get_model_metadata_response_rest(rest_url)
input_metadata_v4, output_metadata_v4 = model_metadata_response(
response=response_v4)
assert model_name == response_v4.model_spec.name
assert expected_input_metadata_v4 == input_metadata_v4
assert expected_output_metadata_v4 == output_metadata_v4
# Model status check
rest_status_url = 'http://localhost:{}/v1/models/resnet'.format(
ports["rest_port"])
status_response = get_model_status_response_rest(rest_status_url)
versions_statuses = status_response.model_version_status
assert len(versions_statuses) == 3
for version_status in versions_statuses:
assert version_status.version in [1, 3, 4]
assert version_status.state == ModelVersionState.AVAILABLE
assert version_status.status.error_code == ErrorCode.OK
assert version_status.status.error_message == _ERROR_MESSAGE[
ModelVersionState.AVAILABLE][ErrorCode.OK]
###
shutil.rmtree(resnet_copy_dir)
shutil.rmtree(resnet_bs4_copy_dir)
shutil.rmtree(resnet_bs8_copy_dir)
time.sleep(10)
def start_server_multi_model(request, get_docker_network, start_minio_server,
get_minio_server_s3, get_image, get_test_dir,
get_docker_context):
shutil.copyfile('tests/functional/config.json',
get_test_dir + '/saved_models/config.json')
AWS_ACCESS_KEY_ID = os.getenv('MINIO_ACCESS_KEY')
AWS_SECRET_ACCESS_KEY = os.getenv('MINIO_SECRET_KEY')
AWS_REGION = os.getenv('AWS_REGION')
client = get_docker_context
network = get_docker_network
_, ports = start_minio_server
grpc_port, rest_port = ports["grpc_port"], ports["rest_port"]
minio_endpoint = 'http://minio.locals3-{}.com:{}'.format(
get_tests_suffix(), grpc_port)
envs = [
'MINIO_ACCESS_KEY=' + AWS_ACCESS_KEY_ID,
'MINIO_SECRET_KEY=' + AWS_SECRET_ACCESS_KEY,
'AWS_ACCESS_KEY_ID=' + AWS_ACCESS_KEY_ID,
'AWS_SECRET_ACCESS_KEY=' + AWS_SECRET_ACCESS_KEY,
'AWS_REGION=' + AWS_REGION, 'S3_ENDPOINT=' + minio_endpoint,
'https_proxy=' + os.getenv('https_proxy', ""),
'no_proxy={}'.format(minio_endpoint)
]
volumes_dict = {
'{}'.format(get_test_dir + '/saved_models/'): {
'bind': '/opt/ml',
'mode': 'ro'
}
}
grpc_port, rest_port = get_ports_for_fixture(port_suffix="07")
command = "/ie-serving-py/start_server.sh ie_serving config " \
"--config_path /opt/ml/config.json --port {} " \
"--rest_port {} --grpc_workers 2 --rest_workers 2".\
format(grpc_port, rest_port)
container = client.containers.run(
image=get_image,
detach=True,
name='ie-serving-py-test-multi-{}'.format(get_tests_suffix()),
ports={
'{}/tcp'.format(grpc_port): grpc_port,
'{}/tcp'.format(rest_port): rest_port
},
remove=True,
volumes=volumes_dict,
environment=envs,
command=command,
network=network.name)
request.addfinalizer(container.kill)
running = wait_endpoint_setup(container)
assert running is True, "docker container was not started successfully"
return container, {"grpc_port": grpc_port, "rest_port": rest_port}
def test_specific_version(self, resnet_multiple_batch_sizes, get_test_dir,
start_server_update_flow_specific,
create_grpc_channel):
_, ports = start_server_update_flow_specific
resnet, resnet_bs4, resnet_bs8 = resnet_multiple_batch_sizes
dir = get_test_dir + '/saved_models/' + 'update-{}/'.format(
get_tests_suffix())
# ensure model dir is empty at the beginning
shutil.rmtree(dir, ignore_errors=True)
stub = create_grpc_channel('localhost:{}'.format(ports["grpc_port"]),
PREDICTION_SERVICE)
status_stub = create_grpc_channel(
'localhost:{}'.format(ports["grpc_port"]), MODEL_SERVICE)
resnet_copy_dir = copy_model(resnet, 1, dir)
resnet_bs4_copy_dir = copy_model(resnet_bs4, 4, dir)
# This could be replaced with status polling
time.sleep(8)
# Available versions: 1, 4
print("Getting info about resnet model")
model_name = 'resnet'
in_name = 'map/TensorArrayStack/TensorArrayGatherV3'
out_name = 'softmax_tensor'
expected_input_metadata_v1 = {
in_name: {
'dtype': 1,
'shape': [1, 3, 224, 224]
}
}
expected_output_metadata_v1 = {
out_name: {
'dtype': 1,
'shape': [1, 1001]
}
}
request = get_model_metadata(model_name=model_name, version=1)
response = stub.GetModelMetadata(request, 10)
input_metadata, output_metadata = model_metadata_response(
response=response)
print(output_metadata)
assert model_name == response.model_spec.name
assert expected_input_metadata_v1 == input_metadata
assert expected_output_metadata_v1 == output_metadata
request_latest = get_model_metadata(model_name=model_name)
response_latest = stub.GetModelMetadata(request_latest, 10)
print("response", response_latest)
input_metadata_latest, output_metadata_latest = \
model_metadata_response(response=response_latest)
request_v4 = get_model_metadata(model_name=model_name, version=4)
response_v4 = stub.GetModelMetadata(request_v4, 10)
print("response", response_v4)
input_metadata_v4, output_metadata_v4 = model_metadata_response(
response=response_latest)
assert response_v4.model_spec.name == response_latest.model_spec.name
assert input_metadata_v4 == input_metadata_latest
assert output_metadata_v4 == output_metadata_latest
# Model status check
model_name = 'resnet'
request = get_model_status(model_name=model_name)
status_response = status_stub.GetModelStatus(request, 10)
versions_statuses = status_response.model_version_status
assert len(versions_statuses) == 2
for version_status in versions_statuses:
assert version_status.version in [1, 4]
assert version_status.state == ModelVersionState.AVAILABLE
assert version_status.status.error_code == ErrorCode.OK
assert version_status.status.error_message == _ERROR_MESSAGE[
ModelVersionState.AVAILABLE][ErrorCode.OK]
###
shutil.rmtree(resnet_bs4_copy_dir)
resnet_bs8_copy_dir = copy_model(resnet_bs8, 3, dir)
time.sleep(10)
# Available versions: 1, 3
request_latest = get_model_metadata(model_name=model_name)
response_latest = stub.GetModelMetadata(request_latest, 10)
print("response", response_latest)
input_metadata_latest, output_metadata_latest = \
model_metadata_response(response=response_latest)
request_v3 = get_model_metadata(model_name=model_name, version=3)
response_v3 = stub.GetModelMetadata(request_v3, 10)
print("response", response_v3)
input_metadata_v3, output_metadata_v3 = model_metadata_response(
response=response_v3)
assert response_v3.model_spec.name == response_latest.model_spec.name
assert input_metadata_v3 == input_metadata_latest
assert output_metadata_v3 == output_metadata_latest
# Model status check
model_name = 'resnet'
request = get_model_status(model_name=model_name)
status_response = status_stub.GetModelStatus(request, 10)
versions_statuses = status_response.model_version_status
assert len(versions_statuses) == 3
for version_status in versions_statuses:
assert version_status.version in [1, 3, 4]
if version_status.version == 4:
assert version_status.state == ModelVersionState.END
assert version_status.status.error_code == ErrorCode.OK
assert version_status.status.error_message == _ERROR_MESSAGE[
ModelVersionState.END][ErrorCode.OK]
elif version_status.version == 1 or version_status.version == 3:
assert version_status.state == ModelVersionState.AVAILABLE
assert version_status.status.error_code == ErrorCode.OK
assert version_status.status.error_message == _ERROR_MESSAGE[
ModelVersionState.AVAILABLE][ErrorCode.OK]
###
# Available versions: 1, 3, 4
resnet_bs4_copy_dir = copy_model(resnet_bs4, 4, dir)
time.sleep(10)
request_v1 = get_model_metadata(model_name=model_name, version=1)
response_v1 = stub.GetModelMetadata(request_v1, 10)
input_metadata_v1, output_metadata_v1 = model_metadata_response(
response=response_v1)
assert model_name == response_v1.model_spec.name
assert expected_input_metadata_v1 == input_metadata_v1
assert expected_output_metadata_v1 == output_metadata_v1
expected_input_metadata_v3 = {
in_name: {
'dtype': 1,
'shape': [8, 3, 224, 224]
}
}
expected_output_metadata_v3 = {
out_name: {
'dtype': 1,
'shape': [8, 1001]
}
}
request_v3 = get_model_metadata(model_name=model_name, version=3)
response_v3 = stub.GetModelMetadata(request_v3, 10)
input_metadata_v3, output_metadata_v3 = model_metadata_response(
response=response_v3)
assert model_name == response_v3.model_spec.name
assert expected_input_metadata_v3 == input_metadata_v3
assert expected_output_metadata_v3 == output_metadata_v3
expected_input_metadata_v4 = {
in_name: {
'dtype': 1,
'shape': [4, 3, 224, 224]
}
}
expected_output_metadata_v4 = {
out_name: {
'dtype': 1,
'shape': [4, 1001]
}
}
request_v4 = get_model_metadata(model_name=model_name)
response_v4 = stub.GetModelMetadata(request_v4, 10)
input_metadata_v4, output_metadata_v4 = model_metadata_response(
response=response_v4)
assert model_name == response_v4.model_spec.name
assert expected_input_metadata_v4 == input_metadata_v4
assert expected_output_metadata_v4 == output_metadata_v4
# Model status check
model_name = 'resnet'
request = get_model_status(model_name=model_name)
status_response = status_stub.GetModelStatus(request, 10)
versions_statuses = status_response.model_version_status
assert len(versions_statuses) == 3
for version_status in versions_statuses:
assert version_status.version in [1, 3, 4]
assert version_status.state == ModelVersionState.AVAILABLE
assert version_status.status.error_code == ErrorCode.OK
assert version_status.status.error_message == _ERROR_MESSAGE[
ModelVersionState.AVAILABLE][ErrorCode.OK]
###
shutil.rmtree(resnet_copy_dir)
shutil.rmtree(resnet_bs4_copy_dir)
shutil.rmtree(resnet_bs8_copy_dir)
time.sleep(10)
def test_update_rest_grpc(self, resnet_multiple_batch_sizes, get_test_dir,
start_server_update_flow_specific,
create_grpc_channel):
_, ports = start_server_update_flow_specific
resnet, resnet_bs4, resnet_bs8 = resnet_multiple_batch_sizes
dir = get_test_dir + '/saved_models/' + 'update-{}/'.format(
get_tests_suffix())
# ensure model dir is empty at the beginning
shutil.rmtree(dir, ignore_errors=True)
stub = create_grpc_channel('localhost:{}'.format(ports["grpc_port"]),
PREDICTION_SERVICE)
resnet_copy_dir = copy_model(resnet, 1, dir)
resnet_bs4_copy_dir = copy_model(resnet_bs4, 4, dir)
time.sleep(8)
# Available versions: 1, 4
print("Getting info about resnet model")
model_name = 'resnet'
in_name = 'map/TensorArrayStack/TensorArrayGatherV3'
out_name = 'softmax_tensor'
expected_input_metadata_v1 = {
in_name: {
'dtype': 1,
'shape': [1, 3, 224, 224]
}
}
expected_output_metadata_v1 = {
out_name: {
'dtype': 1,
'shape': [1, 1001]
}
}
request = get_model_metadata(model_name=model_name, version=1)
response = stub.GetModelMetadata(request, 10)
input_metadata, output_metadata = model_metadata_response(
response=response)
print(output_metadata)
assert model_name == response.model_spec.name
assert expected_input_metadata_v1 == input_metadata
assert expected_output_metadata_v1 == output_metadata
rest_url = 'http://localhost:{}/v1/models/resnet/metadata'.format(
ports["rest_port"])
response_latest = get_model_metadata_response_rest(rest_url)
print("response", response_latest)
input_metadata_latest, output_metadata_latest = \
model_metadata_response(response=response_latest)
request_v4 = get_model_metadata(model_name=model_name, version=4)
response_v4 = stub.GetModelMetadata(request_v4, 10)
print("response", response_v4)
input_metadata_v4, output_metadata_v4 = model_metadata_response(
response=response_latest)
assert response_v4.model_spec.name == response_latest.model_spec.name
assert input_metadata_v4 == input_metadata_latest
assert output_metadata_v4 == output_metadata_latest
shutil.rmtree(resnet_bs4_copy_dir)
resnet_bs8_copy_dir = copy_model(resnet_bs8, 3, dir)
time.sleep(3)
# Available versions: 1, 3
request_latest = get_model_metadata(model_name=model_name)
response_latest = stub.GetModelMetadata(request_latest, 10)
print("response", response_latest)
input_metadata_latest, output_metadata_latest = \
model_metadata_response(response=response_latest)
rest_url = 'http://localhost:{}/v1/models/resnet/versions/3/metadata'.\
format(ports["rest_port"])
response_v3 = get_model_metadata_response_rest(rest_url)
print("response", response_v3)
input_metadata_v3, output_metadata_v3 = model_metadata_response(
response=response_v3)
assert response_v3.model_spec.name == response_latest.model_spec.name
assert input_metadata_v3 == input_metadata_latest
assert output_metadata_v3 == output_metadata_latest
# Available versions: 1, 3, 4
time.sleep(3)
resnet_bs4_copy_dir = copy_model(resnet_bs4, 4, dir)
time.sleep(3)
rest_url = 'http://localhost:{}/v1/models/resnet/versions/1/metadata'.\
format(ports["rest_port"])
response_v1 = get_model_metadata_response_rest(rest_url)
input_metadata_v1, output_metadata_v1 = model_metadata_response(
response=response_v1)
assert model_name == response.model_spec.name
assert expected_input_metadata_v1 == input_metadata_v1
assert expected_output_metadata_v1 == output_metadata_v1
expected_input_metadata_v3 = {
in_name: {
'dtype': 1,
'shape': [8, 3, 224, 224]
}
}
expected_output_metadata_v3 = {
out_name: {
'dtype': 1,
'shape': [8, 1001]
}
}
request_v3 = get_model_metadata(model_name=model_name, version=3)
response_v3 = stub.GetModelMetadata(request_v3, 10)
input_metadata_v3, output_metadata_v3 = model_metadata_response(
response=response_v3)
assert model_name == response.model_spec.name
assert expected_input_metadata_v3 == input_metadata_v3
assert expected_output_metadata_v3 == output_metadata_v3
expected_input_metadata_v4 = {
in_name: {
'dtype': 1,
'shape': [4, 3, 224, 224]
}
}
expected_output_metadata_v4 = {
out_name: {
'dtype': 1,
'shape': [4, 1001]
}
}
rest_url = 'http://localhost:{}/v1/models/resnet/' \
'versions/4/metadata'.format(ports["rest_port"])
response_v4 = get_model_metadata_response_rest(rest_url)
input_metadata_v4, output_metadata_v4 = model_metadata_response(
response=response_v4)
assert model_name == response_v4.model_spec.name
assert expected_input_metadata_v4 == input_metadata_v4
assert expected_output_metadata_v4 == output_metadata_v4
shutil.rmtree(resnet_copy_dir)
shutil.rmtree(resnet_bs4_copy_dir)
shutil.rmtree(resnet_bs8_copy_dir)
def test_latest_version_rest(self, resnet_multiple_batch_sizes,
get_test_dir,
start_server_update_flow_latest):
_, ports = start_server_update_flow_latest
resnet, resnet_bs4, resnet_bs8 = resnet_multiple_batch_sizes
dir = get_test_dir + '/saved_models/' + 'update-{}/'.format(
get_tests_suffix())
# ensure model dir is empty at the beginning
shutil.rmtree(dir, ignore_errors=True)
resnet_copy_dir = copy_model(resnet, 1, dir)
time.sleep(8)
print("Getting info about resnet model")
model_name = 'resnet'
in_name = 'map/TensorArrayStack/TensorArrayGatherV3'
out_name = 'softmax_tensor'
expected_input_metadata_v1 = {
in_name: {
'dtype': 1,
'shape': [1, 3, 224, 224]
}
}
expected_output_metadata_v1 = {
out_name: {
'dtype': 1,
'shape': [1, 1001]
}
}
rest_url = 'http://localhost:{}/v1/models/resnet/metadata'.format(
ports["rest_port"])
response = get_model_metadata_response_rest(rest_url)
input_metadata, output_metadata = model_metadata_response(
response=response)
print(output_metadata)
assert model_name == response.model_spec.name
assert expected_input_metadata_v1 == input_metadata
assert expected_output_metadata_v1 == output_metadata
# Model status check before update
rest_status_url = 'http://localhost:{}/v1/models/resnet'.format(
ports["rest_port"])
status_response = get_model_status_response_rest(rest_status_url)
versions_statuses = status_response.model_version_status
version_status = versions_statuses[0]
assert len(versions_statuses) == 1
assert version_status.version == 1
assert version_status.state == ModelVersionState.AVAILABLE
assert version_status.status.error_code == ErrorCode.OK
assert version_status.status.error_message == _ERROR_MESSAGE[
ModelVersionState.AVAILABLE][ErrorCode.OK]
###
shutil.rmtree(resnet_copy_dir)
resnet_bs4_copy_dir = copy_model(resnet_bs4, 2, dir)
time.sleep(10)
expected_input_metadata = {
in_name: {
'dtype': 1,
'shape': [4, 3, 224, 224]
}
}
expected_output_metadata = {out_name: {'dtype': 1, 'shape': [4, 1001]}}
response = get_model_metadata_response_rest(rest_url)
input_metadata, output_metadata = model_metadata_response(
response=response)
print(output_metadata)
assert model_name == response.model_spec.name
assert expected_input_metadata == input_metadata
assert expected_output_metadata == output_metadata
# Model status check after update
status_response = get_model_status_response_rest(rest_status_url)
versions_statuses = status_response.model_version_status
assert len(versions_statuses) == 2
for version_status in versions_statuses:
assert version_status.version in [1, 2]
if version_status.version == 1:
assert version_status.state == ModelVersionState.END
assert version_status.status.error_code == ErrorCode.OK
assert version_status.status.error_message == _ERROR_MESSAGE[
ModelVersionState.END][ErrorCode.OK]
elif version_status.version == 2:
assert version_status.state == ModelVersionState.AVAILABLE
assert version_status.status.error_code == ErrorCode.OK
assert version_status.status.error_message == _ERROR_MESSAGE[
ModelVersionState.AVAILABLE][ErrorCode.OK]
###
shutil.rmtree(resnet_bs4_copy_dir)
time.sleep(10)
