def main(): begin = time.time() features = prcoess_data() server = '0.0.0.0:9000' channel = grpc.insecure_channel(server) stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) request = predict_pb2.PredictRequest() request.model_spec.name = 'test' request.model_spec.signature_name = 'serving_default' # tensor_input_ids = construct_tensor([1, 64], features.input_ids) # tensor_input_mask = construct_tensor([1, 64], features.input_mask) # tensor_label_ids = construct_tensor([1, 1], [0]) # tensor_segment_ids = construct_tensor([1, 64], features.segment_ids) # # request.inputs['input_ids'].CopyFrom(tensor_input_ids) # request.inputs['input_mask'].CopyFrom(tensor_input_mask) # request.inputs['label_ids'].CopyFrom(tensor_label_ids) # request.inputs['segment_ids'].CopyFrom(tensor_segment_ids) request.inputs['input_ids'].CopyFrom( contrib_util.make_tensor_proto(features.input_ids, shape=[1, 64])) request.inputs['input_mask'].CopyFrom( contrib_util.make_tensor_proto(features.input_mask, shape=[1, 64])) request.inputs['label_ids'].CopyFrom( contrib_util.make_tensor_proto([0], shape=[1, 1])) request.inputs['segment_ids'].CopyFrom( contrib_util.make_tensor_proto(features.segment_ids, shape=[1, 64])) result = stub.Predict(request, 10.0) # 10 secs timeout end = time.time() - begin output = np.array(result.outputs['probabilities'].float_val) print('time {}'.format(end)) print(output)
def process(self, abs_img_dir, image, input_name, model_name, other_k, signature_name, stub, scale): request = predict_pb2.PredictRequest() # 端口里面的名字什么的,设置的第一级为test,第二级为predict_images request.model_spec.name = model_name request.model_spec.signature_name = signature_name # protobuf 序列化并发送请求和接受结果 request.inputs[input_name].CopyFrom( util.make_tensor_proto(image, dtype=tf.string)) if len(other_k) != 0: for key in list(other_k.keys()): name, _type = key.split(':') if _type == 'int': this_type = tf.int32 elif _type == 'float': this_type = tf.float32 request.inputs[name].CopyFrom( util.make_tensor_proto(other_k[key], dtype=this_type)) result = stub.Predict(request, 90.0) # 时限 # 处理返回的信息 results = { 'result': result, 'abs_img_dir': abs_img_dir, 'scale': scale } # os.popen("chmod 777 " + os.path.join(self.dir_path, 'media', abs_img_dir)).readlines() return results
def predict(host, port, hostname, model, signature_name, input_path): # If hostname not set, we assume the host is a valid knative dns. if hostname: host_option = (( 'grpc.ssl_target_name_override', hostname, ), ) else: host_option = None channel = grpc.insecure_channel(target='{host}:{port}'.format(host=host, port=port), options=host_option) stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) with open(input_path) as json_file: data = json.load(json_file) image = data['instances'][0]['image_bytes']['b64'] key = data['instances'][0]['key'] # Call classification model to make prediction request = predict_pb2.PredictRequest() request.model_spec.name = model request.model_spec.signature_name = signature_name image = base64.b64decode(image) request.inputs['image_bytes'].CopyFrom(make_tensor_proto(image, shape=[1])) request.inputs['key'].CopyFrom(make_tensor_proto(key, shape=[1])) result = stub.Predict(request, 10.0) print(result)
def run(host, port, model, signature_name): channel = grpc.insecure_channel('{host}:{port}'.format(host=host, port=port)) stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) start = time.time() # Call classification model to make prediction request = predict_pb2.PredictRequest() request.model_spec.name = model request.model_spec.signature_name = signature_name request.inputs['sepal_length'].CopyFrom( make_tensor_proto(6.8, shape=[1, 1])) request.inputs['sepal_width'].CopyFrom(make_tensor_proto(3.2, shape=[1, 1])) request.inputs['petal_length'].CopyFrom( make_tensor_proto(5.9, shape=[1, 1])) request.inputs['petal_width'].CopyFrom(make_tensor_proto(2.3, shape=[1, 1])) result = stub.Predict(request, 10.0) end = time.time() time_diff = end - start # Reference: # How to access nested values # https://stackoverflow.com/questions/44785847/how-to-retrieve-float-val-from-a-predictresponse-object print(result) print('time elapased: {}'.format(time_diff))
def get_outputs(self, input_data): # Step 0: check the type of input parameters if not isinstance(self.ns.host, str): print("The type of \"host\" must be str (string)!") raise IllegalArgumentException if not re_match("^[0-9localhost.:/]+$", self.ns.host): print("hostport does not match preseted character-set" ) raise IllegalArgumentException if not isinstance(self.ns.port, int): print("The type of \"port\* must be int!") raise IllegalArgumentException if not isinstance(self.ns.model_name, str): print("the type of \"model_name\" must be str (string)!") raise IllegalArgumentException if not re_match("^[0-9A-Za-z_. \-/]+$", self.ns.model_name): print("model_name does not match preseted character-set" ) raise IllegalArgumentException if not isinstance(input_data, dict): print("the type of \"input_data\" must be dict!") raise IllegalArgumentException if (not isinstance(MAX_RESPONSE_TIME, int)) and (not isinstance(MAX_RESPONSE_TIME, float)): print("the type of \"max_response_time\" must be int or float!") raise IllegalArgumentException # Setup connection channel = implementations.insecure_channel(self.ns.host, self.ns.port) stub = prediction_service_pb2.beta_create_PredictionService_stub(channel) # Initialize the request request = predict_pb2.PredictRequest() request.model_spec.name = self.ns.model_name request.model_spec.signature_name = self.ns.model_signature_name #request.model_spec.version = self.ns.model_version_num # Set the input variables of the request for key, value in input_data.items(): if not re_match("^[0-9A-Za-z_. \-/]+$", key): print("model_name does not match preseted character-set" ) raise IllegalArgumentException if isinstance(value, numpy_ndarray): request.inputs[key].CopyFrom(make_tensor_proto(value, shape=list(value.shape))) elif isinstance(value, int) or isinstance(value, float): request.inputs[key].CopyFrom(make_tensor_proto(value) ) else: request.inputs[key].CopyFrom(make_tensor_proto(value, shape=list(value.shape))) # Obtain the result of prediction response = stub.Predict(request, MAX_RESPONSE_TIME) if PRINT_RESPONSE: responseDict = self.print_response(response) return responseDict
def process(self, image): request = predict_pb2.PredictRequest() request.model_spec.name = self.model_spec_name request.inputs['in'].CopyFrom(make_tensor_proto(image, dtype='float32')) request.inputs['min_size'].CopyFrom(make_tensor_proto(self.min_size, dtype='float32')) request.inputs['factor'].CopyFrom(make_tensor_proto(self.factor, dtype='float32')) request.inputs['thresholds'].CopyFrom(make_tensor_proto(self.thresholds)) result = self.stub.Predict(request, 10.0) bbox_np = self.response_to_np(result, 'box') score_np = self.response_to_np(result, 'prob') faces_list = self.extract_faces(image, bbox_np.clip(min=0), score_np, config.IMAGE_SIZE) return faces_list
def send_inference_request(self, image_list): """ input: image should be numpy array """ request = predict_pb2.PredictRequest() request.model_spec.name = self.model_spec_name request.inputs['in'].CopyFrom( make_tensor_proto(image_list, dtype='float32')) request.inputs['phase'].CopyFrom(make_tensor_proto(False, dtype='bool')) result = self.stub.Predict(request, 10.0) # 10.0s timeout embeddings_np = self.response_to_np(result, 'out') return embeddings_np
def main(): # parse command line arguments host, port, image, image_path, batch_mode = parse_args() channel = implementations.insecure_channel(host, int(port)) stub = prediction_service_pb2.beta_create_PredictionService_stub(channel) imagedata = [] if len(image_path) > 0: filenames = [(image_path + '/' + f) for f in listdir(image_path) if isfile(join(image_path, f))] for filename in filenames: with open(filename, 'rb') as f: data = f.read() imagedata.append(data) else: with open(image, 'rb') as f: data = f.read() imagedata.append(data) start = time.time() if batch_mode: print('In batch mode') request = predict_pb2.PredictRequest() request.model_spec.name = 'documents' request.model_spec.signature_name = 'serving_default' request.inputs['key'].CopyFrom( tf.contrib.util.make_tensor_proto("0", shape=[1])) request.inputs['image_bytes'].CopyFrom( make_tensor_proto(imagedata, shape=[len(imagedata)])) result = stub.Predict(request, 60.0) print(result) else: print('In one-by-one mode') for data in imagedata: request = predict_pb2.PredictRequest() request.model_spec.name = 'documents' request.model_spec.signature_name = 'serving_default' request.inputs['key'].CopyFrom( tf.contrib.util.make_tensor_proto("0", shape=[1])) request.inputs['image_bytes'].CopyFrom( make_tensor_proto(data, shape=[1])) result = stub.Predict(request, 60.0) # 60 secs timeout print(result) end = time.time() time_diff = end - start print('time elapased: {}'.format(time_diff))
def main(): # parse command line arguments host, port, image_path, batch_mode = parse_args() channel = implementations.insecure_channel(host, int(port)) stub = prediction_service_pb2.beta_create_PredictionService_stub(channel) filenames = [(image_path + '/' + f) for f in listdir(image_path) if isfile(join(image_path, f))] files = [] imagedata = [] for filename in filenames: f = open(filename, 'rb') files.append(f) data = f.read() imagedata.append(data) start = time.time() if batch_mode: print('In batch mode') request = predict_pb2.PredictRequest() #name of the model trained and exported for our specific purpose #model name will differ based on the model you used (example: use model from tensorflow model zoo and give name as mentioned there) request.model_spec.name = 'mnist' request.model_spec.signature_name = 'predict_images' #signature name as per exporter.py file request.inputs['images'].CopyFrom( make_tensor_proto(imagedata, shape=[len(imagedata)])) result = stub.Predict(request, 60.0) # 60 secs timeout print(result) else: print('In one-by-one mode') for data in imagedata: request = predict_pb2.PredictRequest() request.model_spec.name = 'mnist' request.model_spec.signature_name = 'predict_images' request.inputs['images'].CopyFrom( make_tensor_proto(data, shape=[1])) result = stub.Predict(request, 60.0) # 60 secs timeout print(result) end = time.time() time_diff = end - start print('time elapased: {}'.format(time_diff))
def post(self): json_data = request.get_json(force=True) image_path = json_data['image_path'] model_name = json_data['model_name'] for inx, i in enumerate(dictionary_data): i['model_name'] == model_name break model_data = dictionary_data[inx] signature_name = model_data['signature_name'] inputs = model_data['inputs'] outputs = model_data['outputs'] outputs_type = model_data['outputs_type'] image_data = image_preprocessing(model_name, image_path) # 예측 !! request_tensor.model_spec.name = model_name request_tensor.model_spec.signature_name = signature_name request_tensor.inputs[inputs].CopyFrom( make_tensor_proto(image_data, shape=list(image_data.shape))) result_predict = stub.Predict(request_tensor, 1000.0) print("-----result-----") print(result_predict) final = {} final['결과'] = list( type_val(result_predict, output_name=outputs, output_type=outputs_type)) return final
def get_model_output(data): ''' Call tensorflow model server through grpc for model output ''' host, port, model, signature_name = \ model_params.tensorflow_host, model_params.tensorflow_port, \ model_params.tensorflow_model, model_params.tensorflow_signature_name channel = grpc.insecure_channel('{host}:{port}'.format(host=host, port=port)) stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) start = time.time() # Call classification model to make prediction on the image request = predict_pb2.PredictRequest() request.model_spec.name = model request.model_spec.signature_name = signature_name request.inputs['input'].CopyFrom(make_tensor_proto(data['grid_table'], shape=data['grid_table'].shape)) result = stub.Predict(request, 10.0) outputs_tensor_proto = result.outputs['output'] shape = tf.TensorShape(outputs_tensor_proto.tensor_shape) outputs = np.array(outputs_tensor_proto.float_val).reshape(shape.as_list()) return outputs
def run(inputs): beg = time.time() # 获取输入 pinyin = inputs['PinYin'] # 处理数据 # pinyin = ['jin1','tian1','tian1','qi4','zhen1','hao3'] pinyin = [lm_pinyin_vocab.index(i) for i in pinyin] pinyin = np.asarray(pinyin).reshape(1, -1) # print(pinyin.shape, pinyin) host, port = server.split(':') channel = implementations.insecure_channel(host, int(port)) # stub = prediction_service_pb2.beta_create_PredictionService_stub(channel) stub = prediction_service_pb2_grpc.PredictionServiceStub( channel._channel) request = predict_pb2.PredictRequest() request.model_spec.name = model_name request.model_spec.signature_name = 'predict_Pinyin2Hanzi' request.inputs['pinyin'].CopyFrom( make_tensor_proto(pinyin, shape=pinyin.shape, dtype='int32')) # 1是batch_size result = stub.Predict(request, 60.0) pred_label = np.array(result.outputs['hanzi'].int_val) # print(pred_label.shape, pred_label) hanzi = [lm_hanzi_vocab[i] for i in pred_label] # print( bytes("".join(hanzi), encoding = "utf8")) # 返回 ret = {'HanZi': "".join(hanzi)} print('LM_serving Time:', time.time() - beg) return ret
def main(): # parse command line arguments host, port, test_data = parse_args() channel = implementations.insecure_channel(host, int(port)) stub = prediction_service_pb2.beta_create_PredictionService_stub(channel) # Send request # See prediction_service.proto for gRPC request/response details. start = time.time() request = predict_pb2.PredictRequest() # Call GAN model to make prediction on the image request.model_spec.name = 'base' request.model_spec.signature_name = 'predict' print(test_data.shape) request.inputs['states'].CopyFrom( make_tensor_proto(test_data, dtype=tf.float32)) result = stub.Predict(request, 60.0) # 60 secs timeout end = time.time() time_diff = end - start print(result) print('time elapased: {}'.format(time_diff))
def main(): # parse command line arguments host, port, image = parse_args() channel = implementations.insecure_channel(host, int(port)) stub = prediction_service_pb2.beta_create_PredictionService_stub(channel) # Send request with open(image, 'rb') as f: # See prediction_service.proto for gRPC request/response details. data = f.read() start = time.time() request = predict_pb2.PredictRequest() # Call GAN model to make prediction on the image request.model_spec.name = 'gan' request.model_spec.signature_name = 'predict_images' request.inputs['images'].CopyFrom(make_tensor_proto(data, shape=[1])) result = stub.Predict(request, 60.0) # 60 secs timeout end = time.time() time_diff = end - start print(result) print('time elapased: {}'.format(time_diff))
def main(): # Parse command line arguments host, port = parse_args() # Create a grpc channel using the IP adress and the port channel = implementations.insecure_channel(host, int(port)) # Create a stub stub = prediction_service_pb2.beta_create_PredictionService_stub(channel) # Name of the data that should be send to the server file = '../data/raw/creditcard_dataset_fraud.csv' data = get_data(data_type="test", data_dir=file) sample_df = data.sample(20) for _, row in sample_df.iterrows(): input_data = [[s for s in row]] # print(input_data) # Create a request object request = predict_pb2.PredictRequest() # Name of the model running on the tensorflow_model_server (either locally or in Docker container) request.model_spec.name = 'anamoly_detection' # Name of the defined prediction signature in the SavedModelInstance on the server (either locally or in Docker container) request.model_spec.signature_name = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY # Make a request (time-out after 20 seconds) request.inputs['inputs'].CopyFrom( make_tensor_proto(input_data, shape=[1, 29])) result = stub.Predict(request, 20.0) # 60 secs timeout print(result)
def create_warmup_requests_numpy(self, batch_sizes, export_dir): """Creates warm-up requests for a given feature specification. This writes an output file in `export_dir/assets.extra/tf_serving_warmup_requests` for use with Servo. Args: batch_sizes: Batch sizes of warm-up requests to write. export_dir: Base directory for the export. Returns: The filename written. """ feature_spec = self._get_input_features_for_receiver_fn() flat_feature_spec = tensorspec_utils.flatten_spec_structure( feature_spec) tf.io.gfile.makedirs(export_dir) request_filename = os.path.join(export_dir, 'tf_serving_warmup_requests') with tf.python_io.TFRecordWriter(request_filename) as writer: for batch_size in batch_sizes: request = predict_pb2.PredictRequest() request.model_spec.name = self._model_name numpy_feature_specs = tensorspec_utils.make_constant_numpy( flat_feature_spec, constant_value=0, batch_size=batch_size) for key, numpy_spec in numpy_feature_specs.items(): request.inputs[key].CopyFrom( contrib_util.make_tensor_proto(numpy_spec)) log = prediction_log_pb2.PredictionLog( predict_log=prediction_log_pb2.PredictLog(request=request)) writer.write(log.SerializeToString()) return request_filename
def predict_test1(batch_size, serving_config, img): channel = grpc.insecure_channel(serving_config['hostport'], options=[ ('grpc.max_send_message_length', serving_config['max_message_length']), ('grpc.max_receive_message_length', serving_config['max_message_length']) ]) stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) #img = cv2.imdecode(image, cv2.IMREAD_COLOR) #cv2.imshow("Image", image) # Test on some demo image and visualize output. #image_path = os.path.join(os.getcwd(),"images/") #image_names = sorted(os.listdir(image_path)) #print(image_names) #img = mpimg.imread(image_path + image_names[1]) # Creating random images for given batch size #input_data=np.ones((500,500,3),dtype="uint8") input_data = img print(img.shape) request = predict_pb2.PredictRequest() request.model_spec.name = serving_config['model_name'] request.model_spec.signature_name = serving_config['signature_name'] request.inputs['input0'].CopyFrom( make_tensor_proto(input_data, shape=img.shape, dtype=types_pb2.DT_UINT8)) result = stub.Predict(request, serving_config['timeout']) channel.close() return result
def make_request(image_path, server): """ :param image_path: :param server: :return: """ channel = grpc.insecure_channel(server) stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) image = cv2.imread(image_path, cv2.IMREAD_COLOR) image = cv2.resize(image, (CFG.ARCH.INPUT_SIZE[0], CFG.ARCH.INPUT_SIZE[1]), interpolation=cv2.INTER_LINEAR) image = np.array(image, np.float32) / 127.5 - 1.0 image_list = np.array([image], dtype=np.float32) request = predict_pb2.PredictRequest() request.model_spec.name = 'crnn' request.model_spec.signature_name = 'crnn_prediction_result' request.inputs['input_tensor'].CopyFrom( make_tensor_proto( image_list, shape=[1, CFG.ARCH.INPUT_SIZE[1], CFG.ARCH.INPUT_SIZE[0], 3])) try: result = stub.Predict(request, 10.0) return result except Exception as err: print(err) return None
def run(self, image, width, height): """image: bgr image return (boxes, scores, classes, num_detections) """ request = predict_pb2.PredictRequest() request.model_spec.name = self.model_name request.model_spec.signature_name = self.signature_name request.model_spec.version.value = int(self.version) request.inputs['image'].CopyFrom( make_tensor_proto(image, shape=[1, width, height, 3])) result = self.stub.Predict(request) boxes, scores, classes, num_detections = result.outputs[ 'boxes'].float_val, result.outputs[ 'scores'].float_val, result.outputs[ 'classes'].float_val, result.outputs[ 'num_detections'].float_val num_detections = int(num_detections[0]) boxes = np.reshape(boxes, [1, num_detections, 4]) scores = np.reshape(scores, [1, num_detections]) classes = np.reshape(classes, [1, num_detections]) return boxes, scores, classes, num_detections
def run(host, port, image, model, signature_name): # channel = grpc.insecure_channel('%s:%d' % (host, port)) channel = implementations.insecure_channel(host, port) stub = prediction_service_pb2.beta_create_PredictionService_stub(channel) # Read an image data = imread(image) data = data.astype(np.float32) start = time.time() # Call classification model to make prediction on the image request = predict_pb2.PredictRequest() request.model_spec.name = model request.model_spec.signature_name = signature_name request.inputs['image'].CopyFrom( make_tensor_proto(data, shape=[1, 28, 28, 1])) result = stub.Predict(request, 10.0) end = time.time() time_diff = end - start # Reference: # How to access nested values # https://stackoverflow.com/questions/44785847/how-to-retrieve-float-val-from-a-predictresponse-object print(result) print('time elapased: {}'.format(time_diff))
def main(_): channel = grpc.insecure_channel(FLAGS.server) stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) request = predict_pb2.PredictRequest() request.model_spec.name = FLAGS.model request.model_spec.signature_name = "serving_default" img = cv2.imread(FLAGS.img) retr, buffer = cv2.imencode(".jpg", img) jpg_str = base64.urlsafe_b64encode(buffer).decode("utf-8") difference_sum = 0 for i in range(FLAGS.num_tests): request.inputs['image'].CopyFrom(make_tensor_proto(jpg_str,shape=[1,])) start = time.time() response = stub.Predict(request,10) diff = time.time() - start print("Predict num: {} consumed time: {}".format(i,diff)) difference_sum += diff if FLAGS.vis: classes = np.squeeze(make_ndarray(response.outputs["class"])) box = np.squeeze(make_ndarray(response.outputs["box"])) confidence = np.squeeze(make_ndarray(response.outputs["confidence"])) visimg = cv2.imread(FLAGS.img) for o in range(len(classes)): draw_boxes(visimg,box[o],confidence[o],classes[o]) cv2.imshow("Vis",visimg) cv2.waitKey(1) print("Average time: {}".format(difference_sum / FLAGS.num_tests))
def do_inference(hostport, work_dir, concurrency, num_tests): """Tests PredictionService with concurrent requests. Args: hostport: Host:port address of the PredictionService. work_dir: The full path of working directory for test data set. concurrency: Maximum number of concurrent requests. num_tests: Number of test images to use. Returns: The classification error rate. Raises: IOError: An error occurred processing test data set. """ test_data_set = mnist_input_data.read_data_sets(work_dir).test channel = grpc.insecure_channel(hostport) stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) result_counter = _ResultCounter(num_tests, concurrency) for _ in range(num_tests): request = predict_pb2.PredictRequest() request.model_spec.name = 'mnist' request.model_spec.signature_name = 'predict_images' image, label = test_data_set.next_batch(1) request.inputs['images'].CopyFrom( contrib_util.make_tensor_proto(image[0], shape=[1, image[0].size])) result_counter.throttle() result_future = stub.Predict.future(request, 5.0) # 5 seconds result_future.add_done_callback( _create_rpc_callback(label[0], result_counter)) return result_counter.get_error_rate()
def main(args): host, port = args.host, args.port mnist = input_data.read_data_sets(args.data_dir, one_hot=True) channel = implementations.insecure_channel(host, int(port)) stub = prediction_service_pb2.beta_create_PredictionService_stub(channel) start_time = time.time() counter = 0 num_tests = args.num_tests time for _ in range(num_tests): image, label = mnist.test.next_batch(1) request = predict_pb2.PredictRequest() request.model_spec.name = 'mnist' request.model_spec.signature_name = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY request.inputs['x_input'].CopyFrom( make_tensor_proto(image[0], shape=[1, 28, 28])) result_future = stub.Predict(request, 10.0) exception = result_future.exception() if exception: print(exception) else: response = np.array(result_future.outputs['y_output'].float_val) if np.argmax(response) == np.argmax(label): counter += 1 else: pass print("Accuracy= %0.2f" % ((counter * 1.0 / num_tests) * 100)) print("Time takes to run the test %0.2f" % (time.time() - start_time))
def run(host, port, image_filepath, model, signature_name): channel = grpc.insecure_channel('{host}:{port}'.format(host=host, port=port)) stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) # Read an image data = np.load(image_filepath) data = data.reshape((1, 28, 28, 1)) data = data.astype(np.float32) / 255. start = time.time() # Call classification model to make prediction on the image request = predict_pb2.PredictRequest() request.model_spec.name = model request.model_spec.signature_name = signature_name request.inputs['input'].CopyFrom(make_tensor_proto(data, dtype=np.float32, shape=[1, 28, 28, 1], verify_shape=True)) result = stub.Predict(request, 10.0) end = time.time() time_diff = end - start # Reference: # How to access nested values # https://stackoverflow.com/questions/44785847/how-to-retrieve-float-val-from-a-predictresponse-object print('time elapased: {}'.format(time_diff)) print(result) print('') print('Predicted class: {}'.format(result.outputs['classes'].int64_val)) print('Probabilities: {}'.format(result.outputs['probabilities'].float_val))
def generate_tokens(input_tokens): input_data = np.array([input_tokens], dtype='int32') request.inputs['input'].CopyFrom(make_tensor_proto(input_data, shape=input_data.shape)) response = stub.Predict(request, timeout) tensor = response.outputs['output'] return [tensor.int_val]
def get_fake_request(model_name, data_shape, input_blob, version=None): request = predict_pb2.PredictRequest() request.model_spec.name = model_name if version is not None: request.model_spec.version.value = version data = np.ones(shape=data_shape) request.inputs[input_blob].CopyFrom( make_tensor_proto(data, shape=data.shape)) return request
def get_predictions(host, port, model_name, signature_name, input_placeholder_name, mat): channel = implementations.insecure_channel(host, port)._channel stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) request = predict_pb2.PredictRequest() request.model_spec.name = model_name request.model_spec.signature_name = signature_name request.inputs[input_placeholder_name].CopyFrom(make_tensor_proto(mat)) result = stub.Predict(request) return result
def __create_service_request(self, mr_encoded): """Assembles a request for the prediction service.""" request = predict_pb2.PredictRequest() request.model_spec.name = self.model_name request.inputs['input'].CopyFrom( make_tensor_proto([mr_encoded], shape=[1])) return request
def main(): tokenizer = tokenization.FullTokenizer( vocab_file='./albert_config/vocab.txt', do_lower_case=True) begin = time.time() sentences = [] sentences_1 = ("您好.麻烦您截图全屏辛苦您了.", "麻烦您截图大一点辛苦您了.最好可以全屏.") for i in range(600): sentences.append(sentences_1) input_ids = [] input_mask = [] label_ids = [] segment_ids = [] for i in sentences: features = prcoess_data(i, tokenizer) input_ids.append(features.input_ids) input_mask.append(features.input_mask) label_ids.append([0]) segment_ids.append(features.segment_ids) server = '0.0.0.0:9000' channel = grpc.insecure_channel(server) stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) request = predict_pb2.PredictRequest() request.model_spec.name = 'test' request.model_spec.signature_name = 'serving_default' input_ids = util.make_tensor_proto(input_ids, shape=[len(sentences), 64]) input_mask = util.make_tensor_proto(input_mask, shape=[len(sentences), 64]) label_ids = util.make_tensor_proto(label_ids, shape=[len(sentences), 1]) segment_ids = util.make_tensor_proto(segment_ids, shape=[len(sentences), 64]) request.inputs['input_ids'].CopyFrom(input_ids) request.inputs['input_mask'].CopyFrom(input_mask) request.inputs['label_ids'].CopyFrom(label_ids) request.inputs['segment_ids'].CopyFrom(segment_ids) result = stub.Predict(request, 10.0) # 10 secs timeout end = time.time() - begin output = np.array(result.outputs['probabilities'].float_val) print(len(sentences) / 2) print('time {}'.format(end))
def inference(stub, request, imgs, kwargs): print("Start processing:") print('\tModel name: {}'.format(kwargs['model_name'])) print('\tImages in shape: {}\n'.format(imgs.shape)) processing_times = np.zeros((0), int) batch_size = int(kwargs['batch_size']) if not (kwargs.get('images_number') or kwargs.get('images_number') != 0): iterations = int((imgs.shape[0] // batch_size)) else: iterations = int(kwargs.get('images_number')) iteration = 0 while iteration <= iterations: for x in range(0, imgs.shape[0] - batch_size + 1, batch_size): iteration += 1 if iteration > iterations: break end_batch = x + batch_size if end_batch > imgs.shape[0]: end_batch = imgs.shape[0] batch = imgs[x:end_batch] request.inputs[kwargs['input_name']].CopyFrom( tf_contrib_util.make_tensor_proto(batch, shape=(batch.shape))) start_time = datetime.datetime.now() result = stub.Predict(request, RPC_TIMEOUT) # result includes a dictionary with all model outputs end_time = datetime.datetime.now() if kwargs['output_name'] not in result.outputs: print("Invalid output name", kwargs['output_name']) print("Available outputs:") for Y in result.outputs: print(Y) exit(1) duration = (end_time - start_time).total_seconds() * 1000 processing_times = \ np.append(processing_times, np.array([int(duration)])) output = \ tf_contrib_util.make_ndarray(result.outputs[kwargs['output_name']]) print('Iteration {}; Processing time: {:.2f} ms; speed {:.2f} fps'. format(iteration, round(np.average(duration), 2), round(1000 * batch_size / np.average(duration), 2))) if kwargs['no_imagenet_classes']: continue print('\tImagenet top results in a single batch:') for i in range(output.shape[0]): single_result = output[[i], ...] max_class = np.argmax(single_result) print('\t\t {} image in batch: {}'.format( i + 1, classes.imagenet_classes[max_class])) if kwargs['performance']: get_processing_performance(processing_times, batch_size) return output