def test_run_inference_auto(self, resnet_multiple_batch_sizes, start_server_batch_model_auto, create_grpc_channel): _, ports = start_server_batch_model_auto print("Downloaded model files:", resnet_multiple_batch_sizes) # Connect to grpc service stub = create_grpc_channel('localhost:{}'.format(ports["grpc_port"]), PREDICTION_SERVICE) batch_input = np.ones((6, 3, 224, 224)) in_name = 'map/TensorArrayStack/TensorArrayGatherV3' out_name = 'softmax_tensor' output = infer(batch_input, input_tensor=in_name, grpc_stub=stub, model_spec_name='resnet', model_spec_version=None, output_tensors=[out_name]) print("output shape", output[out_name].shape) assert output[out_name].shape == (6, 1001), ERROR_SHAPE batch_input = np.ones((1, 3, 224, 224)) in_name = 'map/TensorArrayStack/TensorArrayGatherV3' out_name = 'softmax_tensor' output = infer(batch_input, input_tensor=in_name, grpc_stub=stub, model_spec_name='resnet', model_spec_version=None, output_tensors=[out_name]) print("output shape", output[out_name].shape) assert output[out_name].shape == (1, 1001), ERROR_SHAPE
def test_run_inference(self, download_two_model_versions, start_server_multi_model, create_grpc_channel): print("Downloaded model files:", download_two_model_versions) # Connect to grpc service stub = create_grpc_channel('localhost:9001', PREDICTION_SERVICE) face_img = np.ones((1, 3, 300, 300)) pvb_img = np.ones((1, 3, 1024, 1024)) out_name = "detection_out" in_name = "data" output = infer( face_img, input_tensor=in_name, grpc_stub=stub, model_spec_name=self.model_name, model_spec_version=1, # face detection output_tensors=[out_name]) print("output shape", output[out_name].shape) assert output[out_name].shape == (1, 1, 200, 7), \ '{} with version 1 has invalid output'.format(self.model_name) output = infer( pvb_img, input_tensor=in_name, grpc_stub=stub, model_spec_name='pvb_face_multi_version', model_spec_version=None, # PVB detection output_tensors=[out_name]) print("output shape", output[out_name].shape) assert output[out_name].shape == (1, 1, 200, 7), \ '{} with version latest has invalid output'.format(self.model_name)
def test_run_inference(self, input_data_downloader_v1_224, start_server_single_model_from_s3, create_grpc_channel): """ <b>Description</b> Submit request to gRPC interface serving a single resnet model <b>input data</b> - directory with the model in IR format - docker image with ie-serving-py service - input data in numpy format <b>fixtures used</b> - model downloader - input data downloader - service launching <b>Expected results</b> - response contains proper numpy shape """ # Connect to grpc service stub = create_grpc_channel('localhost:9000', PREDICTION_SERVICE) imgs_v1_224 = np.array(input_data_downloader_v1_224) out_name = 'resnet_v1_50/predictions/Reshape_1' for x in range(0, 10): output = infer(imgs_v1_224, slice_number=x, input_tensor='input', grpc_stub=stub, model_spec_name='resnet', model_spec_version=None, output_tensors=[out_name]) print("output shape", output[out_name].shape) assert output[out_name].shape == (1, 1000), ERROR_SHAPE
def test_run_inference(self, resnet_multiple_batch_sizes, start_server_batch_model, create_grpc_channel): """ <b>Description</b> Submit request to gRPC interface serving a single resnet model <b>input data</b> - directory with the model in IR format - docker image with ie-serving-py service <b>fixtures used</b> - model downloader - service launching <b>Expected results</b> - response contains proper numpy shape """ print("Downloaded model files:", resnet_multiple_batch_sizes) # Connect to grpc service stub = create_grpc_channel('localhost:9003', PREDICTION_SERVICE) batch_input = np.ones((8, 3, 224, 224)) in_name = 'map/TensorArrayStack/TensorArrayGatherV3' out_name = 'softmax_tensor' output = infer(batch_input, input_tensor=in_name, grpc_stub=stub, model_spec_name='resnet', model_spec_version=None, output_tensors=[out_name]) print("output shape", output[out_name].shape) assert output[out_name].shape == (8, 1001), ERROR_SHAPE
def test_run_inference_img(self, vehicle_adas_model_downloader, vehicle_adas_data_downloader, start_server_single_vehicle_model, create_grpc_channel): """ <b>Description</b> Submit request to gRPC interface serving a single vehicle model <b>input data</b> - directory with the model in IR format - docker image with ie-serving-py service <b>fixtures used</b> - model downloader - service launching <b>Expected results</b> - response contains proper numpy shape """ _, ports = start_server_single_vehicle_model imgs_path = os.path.join(vehicle_adas_data_downloader, "data", "annotation_val_images") input_img = self.load_image( os.path.join(imgs_path, "image_000015.jpg"), 672, 384) # Connect to grpc service stub = create_grpc_channel('localhost:{}'.format(ports["grpc_port"]), PREDICTION_SERVICE) in_name = 'data' out_name = 'detection_out' output = infer(input_img, input_tensor=in_name, grpc_stub=stub, model_spec_name='vehicle-detection', model_spec_version=None, output_tensors=[out_name]) print("output shape", output[out_name].shape) assert output[out_name].shape == (1, 1, 200, 7), ERROR_SHAPE detections_sum = 0 result = output[out_name] print("result:" + str(result)) LABEL = 1 for i in range(0, 200): detection = result[0, 0, i] print("detection: " + str(detection)) if not detection[LABEL] == 0.0: detections_sum += 1 print("detections_sum= " + str(detections_sum)) assert detections_sum == 19
def run_inference_grpc(self, imgs, out_name, out_shape, is_correct, model_name, stub): if is_correct: output = infer(imgs, input_tensor='data', grpc_stub=stub, model_spec_name=model_name, model_spec_version=None, output_tensors=[out_name]) print("output shape", output[out_name].shape) assert output[out_name].shape == out_shape, \ ERROR_SHAPE else: with pytest.raises(Exception): infer(imgs, input_tensor='data', grpc_stub=stub, model_spec_name=model_name, model_spec_version=None, output_tensors=[out_name])
def test_run_inference(self, age_gender_model_downloader, create_grpc_channel, start_server_with_mapping): """ <b>Description</b> Submit request to gRPC interface serving a single resnet model <b>input data</b> - directory with the model in IR format - docker image with ie-serving-py service <b>fixtures used</b> - model downloader - service launching <b>Expected results</b> - response contains proper numpy shape """ _, ports = start_server_with_mapping print("Downloaded model files:", age_gender_model_downloader) # Connect to grpc service stub = create_grpc_channel('localhost:{}'.format(ports["grpc_port"]), PREDICTION_SERVICE) imgs_v1_224 = np.ones((1, 3, 62, 62)) output = infer(imgs_v1_224, input_tensor='new_key', grpc_stub=stub, model_spec_name='age_gender', model_spec_version=None, output_tensors=['age', 'gender']) print("output shape", output['age'].shape) print("output shape", output['gender'].shape) assert output['age'].shape == (1, 1, 1, 1), ERROR_SHAPE assert output['gender'].shape == (1, 2, 1, 1), ERROR_SHAPE
def test_run_inference(self, resnet_multiple_batch_sizes, start_server_multi_model, create_grpc_channel): _, ports = start_server_multi_model print("Downloaded model files:", resnet_multiple_batch_sizes) # Connect to grpc service stub = create_grpc_channel('localhost:{}'.format(ports["grpc_port"]), PREDICTION_SERVICE) in_name = 'map/TensorArrayStack/TensorArrayGatherV3' out_name = 'softmax_tensor' img = np.ones((1, 3, 224, 224)) print("Starting inference using resnet model") model_name = "resnet" output = infer(img, input_tensor=in_name, grpc_stub=stub, model_spec_name=model_name, model_spec_version=None, output_tensors=[out_name]) print("output shape", output[out_name].shape) assert output[out_name].shape == (1, 1001), ERROR_SHAPE model_name = "resnet_bs4" imgs = np.ones((4, 3, 224, 224)) print("Starting inference using resnet model") output = infer(imgs, input_tensor=in_name, grpc_stub=stub, model_spec_name=model_name, model_spec_version=None, output_tensors=[out_name]) print("output shape", output[out_name].shape) assert output[out_name].shape == (4, 1001), ERROR_SHAPE model_name = "resnet_bs8" imgs = np.ones((8, 3, 224, 224)) print("Starting inference using resnet model") output = infer(imgs, input_tensor=in_name, grpc_stub=stub, model_spec_name=model_name, model_spec_version=None, output_tensors=[out_name]) print("output shape", output[out_name].shape) assert output[out_name].shape == (8, 1001), ERROR_SHAPE output = infer(img, input_tensor=in_name, grpc_stub=stub, model_spec_name='resnet_s3', model_spec_version=None, output_tensors=[out_name]) print("output shape", output[out_name].shape) assert output[out_name].shape == (1, 1001), ERROR_SHAPE in_name = 'input' out_name = 'resnet_v1_50/predictions/Reshape_1' img = np.ones((1, 3, 224, 224)) in_name = 'data' out_name = 'prob' output = infer(img, input_tensor=in_name, grpc_stub=stub, model_spec_name='resnet_gs', model_spec_version=None, output_tensors=[out_name]) print("output shape", output[out_name].shape) assert output[out_name].shape == (1, 1000), ERROR_SHAPE
def test_run_inference(self, download_two_model_versions, input_data_downloader_v1_224, start_server_multi_model, create_grpc_channel): """ <b>Description</b> Execute inference request using gRPC interface with version specified and without version set on the client. When version is not set server should use the latest version model 2 When version 1 is selected the model from folder 1 should be used and model 2 should be ignored <b>input data</b> - directory with the model in IR format - docker image with ie-serving-py service - input data in numpy format <b>fixtures used</b> - model downloader - input data downloader - service launching <b>Expected results</b> - latest model version serves resnet_v2_50 model - [1,1001] output resnet_v2_50/predictions/Reshape_1 - first model version serves resnet_v1_50 model - [1,1000] output resnet_v1_50/predictions/Reshape_1 """ print("Downloaded model files:", download_two_model_versions) # Connect to grpc service stub = create_grpc_channel('localhost:9001', PREDICTION_SERVICE) imgs_v1_224 = np.array(input_data_downloader_v1_224) out_name_v1 = 'resnet_v1_50/predictions/Reshape_1' out_name_v2 = 'resnet_v2_50/predictions/Reshape_1' print("Starting inference using latest version - no version set") for x in range(0, 10): output = infer(imgs_v1_224, slice_number=x, input_tensor='input', grpc_stub=stub, model_spec_name='resnet', model_spec_version=None, output_tensors=[out_name_v2]) print("output shape", output[out_name_v2].shape) assert output[out_name_v2].shape == (1, 1001),\ 'resnet model with version 1 has invalid output' # both model versions use the same input data shape for x in range(0, 10): output = infer(imgs_v1_224, slice_number=x, input_tensor='input', grpc_stub=stub, model_spec_name='resnet', model_spec_version=1, output_tensors=[out_name_v1]) print("output shape", output[out_name_v1].shape) assert output[out_name_v1].shape == (1, 1000),\ 'resnet model with latest version has invalid output'
def test_run_inference_postprocess(self, vehicle_adas_model_downloader, vehicle_adas_data_downloader, start_server_single_vehicle_model, create_grpc_channel): """ <b>Description</b> Submit request to gRPC interface serving a single vehicle model <b>input data</b> - directory with the model in IR format - docker image with ie-serving-py service <b>fixtures used</b> - model downloader - service launching <b>Expected results</b> - response contains proper numpy shape """ _, ports = start_server_single_vehicle_model imgs_path = os.path.join(vehicle_adas_data_downloader, "data", "annotation_val_images") imgs = np.zeros((0, 3, 384, 672), np.dtype('<f')) input_img = self.load_image( os.path.join(imgs_path, "image_000015.jpg"), 672, 384) imgs = np.append(imgs, input_img, axis=0) # Connect to grpc service stub = create_grpc_channel('localhost:{}'.format(ports["grpc_port"]), PREDICTION_SERVICE) in_name = 'data' out_name = 'detection_out' output = infer(imgs, input_tensor=in_name, grpc_stub=stub, model_spec_name='vehicle-detection', model_spec_version=None, output_tensors=[out_name]) print("output shape", output[out_name].shape) assert output[out_name].shape == (1, 1, 200, 7), ERROR_SHAPE sys.path.append( os.path.abspath( os.path.join(os.path.realpath(__file__), '../../../extras/ams_wrapper/'))) from src.api.models.model_builder import ModelBuilder config_path = os.path.abspath( os.path.join( os.path.realpath(__file__), '../../../extras/ams_models/vehicle_detection_adas_model.json') ) model_adas = ModelBuilder.build_model(config_path, 4000) json_response = model_adas.postprocess_inference_output(output) print("json_response= " + str(json_response)) boxes_count = str(json_response).count("box") print("detected boxes:" + str(boxes_count)) assert boxes_count == 19 try: format_check = json.loads(json_response) except Exception as e: print("json loads exception:" + str(e)) assert False print("format_check:" + str(format_check)) assert format_check["subtype"] == "vehicleDetection"
def test_run_inference(self, download_two_models, input_data_downloader_v1_224, input_data_downloader_v3_331, start_server_multi_model, create_grpc_channel): """ <b>Description</b> Execute inference request using gRPC interface hosting multiple models <b>input data</b> - directory with 2 models in IR format - docker image - input data in numpy format <b>fixtures used</b> - model downloader - input data downloader - service launching <b>Expected results</b> - response contains proper numpy shape for both models set in config file: model resnet_v1_50, pnasnet_large - both served models handles appropriate input formats """ print("Downloaded model files:", download_two_models) # Connect to grpc service stub = create_grpc_channel('localhost:9001', PREDICTION_SERVICE) input_data = input_data_downloader_v1_224[:2, :, :, :] print("Starting inference using resnet model") out_name = 'resnet_v1_50/predictions/Reshape_1' for x in range(0, 10): output = infer_batch(input_data, input_tensor='input', grpc_stub=stub, model_spec_name='resnet_V1_50', model_spec_version=None, output_tensors=[out_name]) print("output shape", output[out_name].shape) assert output[out_name].shape == (2, 1000), ERROR_SHAPE imgs_v1_224 = np.array(input_data_downloader_v1_224) out_name = 'resnet_v1_50/predictions/Reshape_1' for x in range(0, 10): output = infer(imgs_v1_224, slice_number=x, input_tensor='input', grpc_stub=stub, model_spec_name='resnet_gs', model_spec_version=None, output_tensors=[out_name]) print("output shape", output[out_name].shape) assert output[out_name].shape == (1, 1000), ERROR_SHAPE out_name = 'resnet_v1_50/predictions/Reshape_1' for x in range(0, 10): output = infer(imgs_v1_224, slice_number=x, input_tensor='input', grpc_stub=stub, model_spec_name='resnet_s3', model_spec_version=None, output_tensors=[out_name]) print("output shape", output[out_name].shape) assert output[out_name].shape == (1, 1000), ERROR_SHAPE input_data = input_data_downloader_v3_331[:4, :, :, :] print("Starting inference using pnasnet_large model") out_name = 'final_layer/predictions' for x in range(0, 10): output = infer_batch(input_data, input_tensor='input', grpc_stub=stub, model_spec_name='pnasnet_large', model_spec_version=None, output_tensors=[out_name]) print("output shape", output[out_name].shape) assert output[out_name].shape == (4, 1001), ERROR_SHAPE