def test_run_inference_rest(self, resnet_multiple_batch_sizes,
start_server_multi_model):
_, ports = start_server_multi_model
print("Downloaded model files:", resnet_multiple_batch_sizes)
img = np.ones((1, 3, 224, 224))
print("Starting inference using resnet model")
in_name = 'map/TensorArrayStack/TensorArrayGatherV3'
out_name = 'softmax_tensor'
model_name = 'resnet'
rest_url = 'http://localhost:{}/v1/models/{}:predict'.format(
ports["rest_port"], model_name)
output = infer_rest(img, input_tensor=in_name, rest_url=rest_url,
output_tensors=[out_name],
request_format='column_name')
print("output shape", output[out_name].shape)
assert output[out_name].shape == (1, 1001), ERROR_SHAPE
imgs = np.ones((4, 3, 224, 224))
model_name = 'resnet_bs4'
rest_url = 'http://localhost:{}/v1/models/{}:predict'.format(
ports["rest_port"], model_name)
output = infer_rest(imgs, input_tensor=in_name, rest_url=rest_url,
output_tensors=[out_name],
request_format='row_noname')
print("output shape", output[out_name].shape)
assert output[out_name].shape == (4, 1001), ERROR_SHAPE
imgs = np.ones((8, 3, 224, 224))
model_name = 'resnet_bs8'
rest_url = 'http://localhost:{}/v1/models/{}:predict'.format(
ports["rest_port"], model_name)
output = infer_rest(imgs, input_tensor=in_name, rest_url=rest_url,
output_tensors=[out_name],
request_format='row_noname')
print("output shape", output[out_name].shape)
assert output[out_name].shape == (8, 1001), ERROR_SHAPE
model_name = 'resnet_s3'
rest_url = 'http://localhost:{}/v1/models/{}:predict'.format(
ports["rest_port"], model_name)
output = infer_rest(img, input_tensor=in_name, rest_url=rest_url,
output_tensors=[out_name],
request_format='row_name')
print("output shape", output[out_name].shape)
assert output[out_name].shape == (1, 1001), ERROR_SHAPE
in_name = 'input'
out_name = 'resnet_v1_50/predictions/Reshape_1'
model_name = 'resnet_gs'
rest_url = 'http://localhost:{}/v1/models/{}:predict'.format(
ports["rest_port"], model_name)
output = infer_rest(img, input_tensor=in_name, rest_url=rest_url,
output_tensors=[out_name],
request_format='column_noname')
print("output shape", output[out_name].shape)
assert output[out_name].shape == (1, 1000), ERROR_SHAPE
def test_run_inference_rest(self, download_two_model_versions,
start_server_multi_model):
_, ports = start_server_multi_model
print("Downloaded model files:", download_two_model_versions)
face_img = np.ones((1, 3, 300, 300))
pvb_img = np.ones((1, 3, 1024, 1024))
out_name = "detection_out"
in_name = "data"
rest_url = 'http://localhost:{}/v1/models/{}' \
'/versions/1:predict'.format(ports["rest_port"],
self.model_name)
output = infer_rest(face_img,
input_tensor=in_name,
rest_url=rest_url,
output_tensors=[out_name],
request_format='column_name')
print("output shape", output[out_name].shape)
assert output[out_name].shape == (1, 1, 200, 7), \
'{} with version 1 has invalid output'.format(self.model_name)
rest_url = 'http://localhost:{}/v1/models/{}:predict'.format(
ports["rest_port"], self.model_name)
output = infer_rest(pvb_img,
input_tensor=in_name,
rest_url=rest_url,
output_tensors=[out_name],
request_format='column_name')
print("output shape", output[out_name].shape)
assert output[out_name].shape == (1, 1, 200, 7), \
'{} with version latest has invalid output'.format(self.model_name)
def test_run_inference_rest(self, download_two_model_versions,
input_data_downloader_v1_224,
start_server_multi_model):
"""
<b>Description</b>
Execute inference request using REST API interface with version
specified and without version set on the client.
When version is not set server should use the latest version model 2
When version 1 is selected the model from folder 1 should be used
and model 2 should be ignored
<b>input data</b>
- directory with the model in IR format
- docker image with ie-serving-py service
- input data in numpy format
<b>fixtures used</b>
- model downloader
- input data downloader
- service launching
<b>Expected results</b>
- latest model version serves resnet_v2_50 model - [1,1001]
output resnet_v2_50/predictions/Reshape_1
- first model version serves resnet_v1_50 model - [1,1000]
output resnet_v1_50/predictions/Reshape_1
"""
print("Downloaded model files:", download_two_model_versions)
imgs_v1_224 = np.array(input_data_downloader_v1_224)
out_name_v1 = 'resnet_v1_50/predictions/Reshape_1'
out_name_v2 = 'resnet_v2_50/predictions/Reshape_1'
print("Starting inference using latest version - no version set")
rest_url = 'http://localhost:5561/v1/models/resnet:predict'
for x in range(0, 10):
output = infer_rest(imgs_v1_224,
slice_number=x,
input_tensor='input',
rest_url=rest_url,
output_tensors=[out_name_v2],
request_format='column_name')
print("output shape", output[out_name_v2].shape)
assert output[out_name_v2].shape == (1, 1001), \
'resnet model with version 1 has invalid output'
# both model versions use the same input data shape
rest_url = 'http://localhost:5561/v1/models/resnet/versions/1:predict'
for x in range(0, 10):
output = infer_rest(imgs_v1_224,
slice_number=x,
input_tensor='input',
rest_url=rest_url,
output_tensors=[out_name_v1],
request_format='column_name')
print("output shape", output[out_name_v1].shape)
assert output[out_name_v1].shape == (1, 1000), \
'resnet model with latest version has invalid output'
def test_run_inference_rest(self, age_gender_model_downloader,
start_server_with_mapping, request_format):
"""
<b>Description</b>
Submit request to REST API interface serving a single resnet model
<b>input data</b>
- directory with the model in IR format
- docker image with ie-serving-py service
<b>fixtures used</b>
- model downloader
- service launching
<b>Expected results</b>
- response contains proper numpy shape
"""
print("Downloaded model files:", age_gender_model_downloader)
imgs_v1_224 = np.ones((1, 3, 62, 62))
rest_url = 'http://localhost:5556/v1/models/age_gender:predict'
output = infer_rest(imgs_v1_224,
input_tensor='new_key',
rest_url=rest_url,
output_tensors=['age', 'gender'],
request_format=request_format)
print("output shape", output['age'].shape)
print("output shape", output['gender'].shape)
print(output)
assert output['age'].shape == (1, 1, 1, 1), ERROR_SHAPE
assert output['gender'].shape == (1, 2, 1, 1), ERROR_SHAPE
def test_run_inference_rest(self, resnet_multiple_batch_sizes,
start_server_single_model, request_format):
"""
<b>Description</b>
Submit request to REST API interface serving a single resnet model
<b>input data</b>
- directory with the model in IR format
- docker image with ie-serving-py service
<b>fixtures used</b>
- model downloader
- service launching
<b>Expected results</b>
- response contains proper numpy shape
"""
print("Downloaded model files:", resnet_multiple_batch_sizes)
imgs_v1_224 = np.ones((1, 3, 224, 224))
in_name = 'map/TensorArrayStack/TensorArrayGatherV3'
out_name = 'softmax_tensor'
rest_url = 'http://localhost:5555/v1/models/resnet:predict'
output = infer_rest(imgs_v1_224,
input_tensor=in_name,
rest_url=rest_url,
output_tensors=[out_name],
request_format=request_format)
print("output shape", output[out_name].shape)
assert output[out_name].shape == (1, 1001), ERROR_SHAPE
def test_run_inference_rest(self, resnet_v1_50_model_downloader,
input_data_downloader_v1_224,
start_server_single_model,
request_format):
"""
<b>Description</b>
Submit request to REST API interface serving a single resnet model
<b>input data</b>
- directory with the model in IR format
- docker image with ie-serving-py service
- input data in numpy format
<b>fixtures used</b>
- model downloader
- input data downloader
- service launching
<b>Expected results</b>
- response contains proper numpy shape
"""
print("Downloaded model files:", resnet_v1_50_model_downloader)
imgs_v1_224 = np.array(input_data_downloader_v1_224)
out_name = 'resnet_v1_50/predictions/Reshape_1'
rest_url = 'http://localhost:5555/v1/models/resnet:predict'
for x in range(0, 10):
output = infer_rest(imgs_v1_224, slice_number=x,
input_tensor='input', rest_url=rest_url,
output_tensors=[out_name],
request_format=request_format)
print("output shape", output[out_name].shape)
assert output[out_name].shape == (1, 1000), ERROR_SHAPE
def test_run_inference_bs4_rest(self, age_gender_model_downloader,
start_server_batch_model_auto_bs4_2out,
request_format):
"""
<b>Description</b>
Submit request to REST API interface serving
a single age-gender model with 2 outputs.
Parameter batch_size explicitly set to 4.
<b>input data</b>
- directory with the model in IR format
- docker image with ie-serving-py service
<b>fixtures used</b>
- model downloader
- service launching
<b>Expected results</b>
- response contains proper numpy shape
"""
print("Downloaded model files:", age_gender_model_downloader)
batch_input = np.ones((4, 3, 62, 62))
in_name = 'data'
out_names = ['age_conv3', 'prob']
rest_url = 'http://localhost:5562/v1/models/age_gender:predict'
output = infer_rest(batch_input,
input_tensor=in_name,
rest_url=rest_url,
output_tensors=out_names,
request_format=request_format)
assert output[out_names[0]].shape == (4, 1, 1, 1), ERROR_SHAPE
assert output[out_names[1]].shape == (4, 2, 1, 1), ERROR_SHAPE
def run_inference_rest(self, imgs, out_name, out_shape, is_correct,
request_format, rest_url):
if is_correct:
output = infer_rest(imgs,
input_tensor='data',
rest_url=rest_url,
output_tensors=[out_name],
request_format=request_format)
print("output shape", output[out_name].shape)
assert output[out_name].shape == out_shape, \
ERROR_SHAPE
else:
output = infer_rest(imgs,
input_tensor='data',
rest_url=rest_url,
output_tensors=[out_name],
request_format=request_format)
assert not output
def test_run_inference_rest(self, download_two_models,
input_data_downloader_v1_224,
input_data_downloader_v3_331,
start_server_multi_model):
"""
<b>Description</b>
Execute inference request using REST API interface hosting multiple
models
<b>input data</b>
- directory with 2 models in IR format
- docker image
- input data in numpy format
<b>fixtures used</b>
- model downloader
- input data downloader
- service launching
<b>Expected results</b>
- response contains proper numpy shape for both models set in config
file: model resnet_v1_50, pnasnet_large
- both served models handles appropriate input formats
"""
print("Downloaded model files:", download_two_models)
input_data = input_data_downloader_v1_224[:2, :, :, :]
print("Starting inference using resnet model")
out_name = 'resnet_v1_50/predictions/Reshape_1'
rest_url = 'http://localhost:5561/v1/models/resnet_V1_50:predict'
for x in range(0, 10):
output = infer_batch_rest(input_data,
input_tensor='input',
rest_url=rest_url,
output_tensors=[out_name],
request_format='column_name')
print("output shape", output[out_name].shape)
assert output[out_name].shape == (2, 1000), ERROR_SHAPE
imgs_v1_224 = np.array(input_data_downloader_v1_224)
out_name = 'resnet_v1_50/predictions/Reshape_1'
rest_url = 'http://localhost:5561/v1/models/resnet_gs:predict'
for x in range(0, 10):
output = infer_rest(imgs_v1_224,
slice_number=x,
input_tensor='input',
rest_url=rest_url,
output_tensors=[out_name],
request_format='column_noname')
print("output shape", output[out_name].shape)
assert output[out_name].shape == (1, 1000), ERROR_SHAPE
out_name = 'resnet_v1_50/predictions/Reshape_1'
rest_url = 'http://localhost:5561/v1/models/resnet_s3:predict'
for x in range(0, 10):
output = infer_rest(imgs_v1_224,
slice_number=x,
input_tensor='input',
rest_url=rest_url,
output_tensors=[out_name],
request_format='row_name')
print("output shape", output[out_name].shape)
assert output[out_name].shape == (1, 1000), ERROR_SHAPE
input_data = input_data_downloader_v3_331[:4, :, :, :]
print("Starting inference using pnasnet_large model")
out_name = 'final_layer/predictions'
rest_url = 'http://localhost:5561/v1/models/pnasnet_large:predict'
for x in range(0, 10):
output = infer_batch_rest(input_data,
input_tensor='input',
rest_url=rest_url,
output_tensors=[out_name],
request_format='row_noname')
print("output shape", output[out_name].shape)
assert output[out_name].shape == (4, 1001), ERROR_SHAPE