def test_run_inference_auto(self, resnet_multiple_batch_sizes,
start_server_batch_model_auto,
create_grpc_channel):
_, ports = start_server_batch_model_auto
print("Downloaded model files:", resnet_multiple_batch_sizes)
# Connect to grpc service
stub = create_grpc_channel('localhost:{}'.format(ports["grpc_port"]),
PREDICTION_SERVICE)
batch_input = np.ones((6, 3, 224, 224))
in_name = 'map/TensorArrayStack/TensorArrayGatherV3'
out_name = 'softmax_tensor'
output = infer(batch_input,
input_tensor=in_name,
grpc_stub=stub,
model_spec_name='resnet',
model_spec_version=None,
output_tensors=[out_name])
print("output shape", output[out_name].shape)
assert output[out_name].shape == (6, 1001), ERROR_SHAPE
batch_input = np.ones((1, 3, 224, 224))
in_name = 'map/TensorArrayStack/TensorArrayGatherV3'
out_name = 'softmax_tensor'
output = infer(batch_input,
input_tensor=in_name,
grpc_stub=stub,
model_spec_name='resnet',
model_spec_version=None,
output_tensors=[out_name])
print("output shape", output[out_name].shape)
assert output[out_name].shape == (1, 1001), ERROR_SHAPE
def test_run_inference(self, download_two_model_versions,
start_server_multi_model, create_grpc_channel):
print("Downloaded model files:", download_two_model_versions)
# Connect to grpc service
stub = create_grpc_channel('localhost:9001', PREDICTION_SERVICE)
face_img = np.ones((1, 3, 300, 300))
pvb_img = np.ones((1, 3, 1024, 1024))
out_name = "detection_out"
in_name = "data"
output = infer(
face_img,
input_tensor=in_name,
grpc_stub=stub,
model_spec_name=self.model_name,
model_spec_version=1, # face detection
output_tensors=[out_name])
print("output shape", output[out_name].shape)
assert output[out_name].shape == (1, 1, 200, 7), \
'{} with version 1 has invalid output'.format(self.model_name)
output = infer(
pvb_img,
input_tensor=in_name,
grpc_stub=stub,
model_spec_name='pvb_face_multi_version',
model_spec_version=None, # PVB detection
output_tensors=[out_name])
print("output shape", output[out_name].shape)
assert output[out_name].shape == (1, 1, 200, 7), \
'{} with version latest has invalid output'.format(self.model_name)
def test_run_inference(self, input_data_downloader_v1_224,
start_server_single_model_from_s3,
create_grpc_channel):
"""
<b>Description</b>
Submit request to gRPC interface serving a single resnet model
<b>input data</b>
- directory with the model in IR format
- docker image with ie-serving-py service
- input data in numpy format
<b>fixtures used</b>
- model downloader
- input data downloader
- service launching
<b>Expected results</b>
- response contains proper numpy shape
"""
# Connect to grpc service
stub = create_grpc_channel('localhost:9000', PREDICTION_SERVICE)
imgs_v1_224 = np.array(input_data_downloader_v1_224)
out_name = 'resnet_v1_50/predictions/Reshape_1'
for x in range(0, 10):
output = infer(imgs_v1_224, slice_number=x,
input_tensor='input', grpc_stub=stub,
model_spec_name='resnet',
model_spec_version=None,
output_tensors=[out_name])
print("output shape", output[out_name].shape)
assert output[out_name].shape == (1, 1000), ERROR_SHAPE
def test_run_inference(self, resnet_multiple_batch_sizes,
start_server_batch_model, create_grpc_channel):
"""
<b>Description</b>
Submit request to gRPC interface serving a single resnet model
<b>input data</b>
- directory with the model in IR format
- docker image with ie-serving-py service
<b>fixtures used</b>
- model downloader
- service launching
<b>Expected results</b>
- response contains proper numpy shape
"""
print("Downloaded model files:", resnet_multiple_batch_sizes)
# Connect to grpc service
stub = create_grpc_channel('localhost:9003', PREDICTION_SERVICE)
batch_input = np.ones((8, 3, 224, 224))
in_name = 'map/TensorArrayStack/TensorArrayGatherV3'
out_name = 'softmax_tensor'
output = infer(batch_input,
input_tensor=in_name,
grpc_stub=stub,
model_spec_name='resnet',
model_spec_version=None,
output_tensors=[out_name])
print("output shape", output[out_name].shape)
assert output[out_name].shape == (8, 1001), ERROR_SHAPE
def test_run_inference_img(self, vehicle_adas_model_downloader,
vehicle_adas_data_downloader,
start_server_single_vehicle_model,
create_grpc_channel):
"""
<b>Description</b>
Submit request to gRPC interface serving a single vehicle model
<b>input data</b>
- directory with the model in IR format
- docker image with ie-serving-py service
<b>fixtures used</b>
- model downloader
- service launching
<b>Expected results</b>
- response contains proper numpy shape
"""
_, ports = start_server_single_vehicle_model
imgs_path = os.path.join(vehicle_adas_data_downloader, "data",
"annotation_val_images")
input_img = self.load_image(
os.path.join(imgs_path, "image_000015.jpg"), 672, 384)
# Connect to grpc service
stub = create_grpc_channel('localhost:{}'.format(ports["grpc_port"]),
PREDICTION_SERVICE)
in_name = 'data'
out_name = 'detection_out'
output = infer(input_img,
input_tensor=in_name,
grpc_stub=stub,
model_spec_name='vehicle-detection',
model_spec_version=None,
output_tensors=[out_name])
print("output shape", output[out_name].shape)
assert output[out_name].shape == (1, 1, 200, 7), ERROR_SHAPE
detections_sum = 0
result = output[out_name]
print("result:" + str(result))
LABEL = 1
for i in range(0, 200):
detection = result[0, 0, i]
print("detection: " + str(detection))
if not detection[LABEL] == 0.0:
detections_sum += 1
print("detections_sum= " + str(detections_sum))
assert detections_sum == 19
def run_inference_grpc(self, imgs, out_name, out_shape, is_correct,
model_name, stub):
if is_correct:
output = infer(imgs,
input_tensor='data',
grpc_stub=stub,
model_spec_name=model_name,
model_spec_version=None,
output_tensors=[out_name])
print("output shape", output[out_name].shape)
assert output[out_name].shape == out_shape, \
ERROR_SHAPE
else:
with pytest.raises(Exception):
infer(imgs,
input_tensor='data',
grpc_stub=stub,
model_spec_name=model_name,
model_spec_version=None,
output_tensors=[out_name])
def test_run_inference(self, age_gender_model_downloader,
create_grpc_channel,
start_server_with_mapping):
"""
<b>Description</b>
Submit request to gRPC interface serving a single resnet model
<b>input data</b>
- directory with the model in IR format
- docker image with ie-serving-py service
<b>fixtures used</b>
- model downloader
- service launching
<b>Expected results</b>
- response contains proper numpy shape
"""
_, ports = start_server_with_mapping
print("Downloaded model files:", age_gender_model_downloader)
# Connect to grpc service
stub = create_grpc_channel('localhost:{}'.format(ports["grpc_port"]),
PREDICTION_SERVICE)
imgs_v1_224 = np.ones((1, 3, 62, 62))
output = infer(imgs_v1_224, input_tensor='new_key', grpc_stub=stub,
model_spec_name='age_gender',
model_spec_version=None,
output_tensors=['age', 'gender'])
print("output shape", output['age'].shape)
print("output shape", output['gender'].shape)
assert output['age'].shape == (1, 1, 1, 1), ERROR_SHAPE
assert output['gender'].shape == (1, 2, 1, 1), ERROR_SHAPE
def test_run_inference(self, resnet_multiple_batch_sizes,
start_server_multi_model,
create_grpc_channel):
_, ports = start_server_multi_model
print("Downloaded model files:", resnet_multiple_batch_sizes)
# Connect to grpc service
stub = create_grpc_channel('localhost:{}'.format(ports["grpc_port"]),
PREDICTION_SERVICE)
in_name = 'map/TensorArrayStack/TensorArrayGatherV3'
out_name = 'softmax_tensor'
img = np.ones((1, 3, 224, 224))
print("Starting inference using resnet model")
model_name = "resnet"
output = infer(img, input_tensor=in_name,
grpc_stub=stub,
model_spec_name=model_name,
model_spec_version=None,
output_tensors=[out_name])
print("output shape", output[out_name].shape)
assert output[out_name].shape == (1, 1001), ERROR_SHAPE
model_name = "resnet_bs4"
imgs = np.ones((4, 3, 224, 224))
print("Starting inference using resnet model")
output = infer(imgs, input_tensor=in_name,
grpc_stub=stub,
model_spec_name=model_name,
model_spec_version=None,
output_tensors=[out_name])
print("output shape", output[out_name].shape)
assert output[out_name].shape == (4, 1001), ERROR_SHAPE
model_name = "resnet_bs8"
imgs = np.ones((8, 3, 224, 224))
print("Starting inference using resnet model")
output = infer(imgs, input_tensor=in_name,
grpc_stub=stub,
model_spec_name=model_name,
model_spec_version=None,
output_tensors=[out_name])
print("output shape", output[out_name].shape)
assert output[out_name].shape == (8, 1001), ERROR_SHAPE
output = infer(img, input_tensor=in_name, grpc_stub=stub,
model_spec_name='resnet_s3',
model_spec_version=None,
output_tensors=[out_name])
print("output shape", output[out_name].shape)
assert output[out_name].shape == (1, 1001), ERROR_SHAPE
in_name = 'input'
out_name = 'resnet_v1_50/predictions/Reshape_1'
img = np.ones((1, 3, 224, 224))
in_name = 'data'
out_name = 'prob'
output = infer(img, input_tensor=in_name, grpc_stub=stub,
model_spec_name='resnet_gs',
model_spec_version=None,
output_tensors=[out_name])
print("output shape", output[out_name].shape)
assert output[out_name].shape == (1, 1000), ERROR_SHAPE
def test_run_inference(self, download_two_model_versions,
input_data_downloader_v1_224,
start_server_multi_model, create_grpc_channel):
"""
<b>Description</b>
Execute inference request using gRPC interface with version specified
and without version set on the client.
When version is not set server should use the latest version model 2
When version 1 is selected the model from folder 1 should be used
and model 2 should be ignored
<b>input data</b>
- directory with the model in IR format
- docker image with ie-serving-py service
- input data in numpy format
<b>fixtures used</b>
- model downloader
- input data downloader
- service launching
<b>Expected results</b>
- latest model version serves resnet_v2_50 model - [1,1001]
output resnet_v2_50/predictions/Reshape_1
- first model version serves resnet_v1_50 model - [1,1000]
output resnet_v1_50/predictions/Reshape_1
"""
print("Downloaded model files:", download_two_model_versions)
# Connect to grpc service
stub = create_grpc_channel('localhost:9001', PREDICTION_SERVICE)
imgs_v1_224 = np.array(input_data_downloader_v1_224)
out_name_v1 = 'resnet_v1_50/predictions/Reshape_1'
out_name_v2 = 'resnet_v2_50/predictions/Reshape_1'
print("Starting inference using latest version - no version set")
for x in range(0, 10):
output = infer(imgs_v1_224,
slice_number=x,
input_tensor='input',
grpc_stub=stub,
model_spec_name='resnet',
model_spec_version=None,
output_tensors=[out_name_v2])
print("output shape", output[out_name_v2].shape)
assert output[out_name_v2].shape == (1, 1001),\
'resnet model with version 1 has invalid output'
# both model versions use the same input data shape
for x in range(0, 10):
output = infer(imgs_v1_224,
slice_number=x,
input_tensor='input',
grpc_stub=stub,
model_spec_name='resnet',
model_spec_version=1,
output_tensors=[out_name_v1])
print("output shape", output[out_name_v1].shape)
assert output[out_name_v1].shape == (1, 1000),\
'resnet model with latest version has invalid output'
def test_run_inference_postprocess(self, vehicle_adas_model_downloader,
vehicle_adas_data_downloader,
start_server_single_vehicle_model,
create_grpc_channel):
"""
<b>Description</b>
Submit request to gRPC interface serving a single vehicle model
<b>input data</b>
- directory with the model in IR format
- docker image with ie-serving-py service
<b>fixtures used</b>
- model downloader
- service launching
<b>Expected results</b>
- response contains proper numpy shape
"""
_, ports = start_server_single_vehicle_model
imgs_path = os.path.join(vehicle_adas_data_downloader, "data",
"annotation_val_images")
imgs = np.zeros((0, 3, 384, 672), np.dtype('<f'))
input_img = self.load_image(
os.path.join(imgs_path, "image_000015.jpg"), 672, 384)
imgs = np.append(imgs, input_img, axis=0)
# Connect to grpc service
stub = create_grpc_channel('localhost:{}'.format(ports["grpc_port"]),
PREDICTION_SERVICE)
in_name = 'data'
out_name = 'detection_out'
output = infer(imgs,
input_tensor=in_name,
grpc_stub=stub,
model_spec_name='vehicle-detection',
model_spec_version=None,
output_tensors=[out_name])
print("output shape", output[out_name].shape)
assert output[out_name].shape == (1, 1, 200, 7), ERROR_SHAPE
sys.path.append(
os.path.abspath(
os.path.join(os.path.realpath(__file__),
'../../../extras/ams_wrapper/')))
from src.api.models.model_builder import ModelBuilder
config_path = os.path.abspath(
os.path.join(
os.path.realpath(__file__),
'../../../extras/ams_models/vehicle_detection_adas_model.json')
)
model_adas = ModelBuilder.build_model(config_path, 4000)
json_response = model_adas.postprocess_inference_output(output)
print("json_response= " + str(json_response))
boxes_count = str(json_response).count("box")
print("detected boxes:" + str(boxes_count))
assert boxes_count == 19
try:
format_check = json.loads(json_response)
except Exception as e:
print("json loads exception:" + str(e))
assert False
print("format_check:" + str(format_check))
assert format_check["subtype"] == "vehicleDetection"
def test_run_inference(self, download_two_models,
input_data_downloader_v1_224,
input_data_downloader_v3_331,
start_server_multi_model, create_grpc_channel):
"""
<b>Description</b>
Execute inference request using gRPC interface hosting multiple models
<b>input data</b>
- directory with 2 models in IR format
- docker image
- input data in numpy format
<b>fixtures used</b>
- model downloader
- input data downloader
- service launching
<b>Expected results</b>
- response contains proper numpy shape for both models set in config
file: model resnet_v1_50, pnasnet_large
- both served models handles appropriate input formats
"""
print("Downloaded model files:", download_two_models)
# Connect to grpc service
stub = create_grpc_channel('localhost:9001', PREDICTION_SERVICE)
input_data = input_data_downloader_v1_224[:2, :, :, :]
print("Starting inference using resnet model")
out_name = 'resnet_v1_50/predictions/Reshape_1'
for x in range(0, 10):
output = infer_batch(input_data,
input_tensor='input',
grpc_stub=stub,
model_spec_name='resnet_V1_50',
model_spec_version=None,
output_tensors=[out_name])
print("output shape", output[out_name].shape)
assert output[out_name].shape == (2, 1000), ERROR_SHAPE
imgs_v1_224 = np.array(input_data_downloader_v1_224)
out_name = 'resnet_v1_50/predictions/Reshape_1'
for x in range(0, 10):
output = infer(imgs_v1_224,
slice_number=x,
input_tensor='input',
grpc_stub=stub,
model_spec_name='resnet_gs',
model_spec_version=None,
output_tensors=[out_name])
print("output shape", output[out_name].shape)
assert output[out_name].shape == (1, 1000), ERROR_SHAPE
out_name = 'resnet_v1_50/predictions/Reshape_1'
for x in range(0, 10):
output = infer(imgs_v1_224,
slice_number=x,
input_tensor='input',
grpc_stub=stub,
model_spec_name='resnet_s3',
model_spec_version=None,
output_tensors=[out_name])
print("output shape", output[out_name].shape)
assert output[out_name].shape == (1, 1000), ERROR_SHAPE
input_data = input_data_downloader_v3_331[:4, :, :, :]
print("Starting inference using pnasnet_large model")
out_name = 'final_layer/predictions'
for x in range(0, 10):
output = infer_batch(input_data,
input_tensor='input',
grpc_stub=stub,
model_spec_name='pnasnet_large',
model_spec_version=None,
output_tensors=[out_name])
print("output shape", output[out_name].shape)
assert output[out_name].shape == (4, 1001), ERROR_SHAPE
