def do_inference(hostport, work_dir, concurrency, num_tests): """Tests PredictionService with concurrent requests. Args: hostport: Host:port address of the PredictionService. work_dir: The full path of working directory for test data set. concurrency: Maximum number of concurrent requests. num_tests: Number of test images to use. Returns: The classification error rate. Raises: IOError: An error occurred processing test data set. """ #test_data_set = mnist_input_data.read_data_sets(work_dir).test test_data_set = DataReadAndNegSamp(file_input='./20190623.ID.test') test_data_set = test_data_set.train_data.values print('test_data_set.shape:', test_data_set.shape, type(test_data_set)) channel = grpc.insecure_channel(hostport) stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) pred_res = [] real_res = [] for index in range(int(num_tests / concurrency)): request = predict_pb2.PredictRequest() request.model_spec.name = 'seq_model' request.model_spec.signature_name = 'pred_class' #request.model_spec.signature_name = 'pred_prob' ## here used 'input' cur_data = test_data_set[index * concurrency:(index + 1) * concurrency] cur_label = cur_data[:, 0] cur_feature = cur_data[:, 1:].reshape((-1, 13)) #print ('cur_data.shape:', cur_data.shape, type(cur_data)) #print ('cur_label.shape:', cur_label.shape, type(cur_label)) #print ('cur_feature.shape:', cur_feature.shape, type(cur_feature)) #image, label = test_data_set[index] #request.inputs['input'].CopyFrom( request.inputs['input_'].CopyFrom( tf.contrib.util.make_tensor_proto(values=cur_feature, dtype=tf.int32, shape=cur_feature.shape)) #tf.contrib.util.make_tensor_proto(image[0], shape=[1, image[0].size])) response = stub.Predict(request, 5.0) #print ('respose:', response, type(response)) results = tf.contrib.util.make_ndarray(response.outputs['output_']) pred_res.append(list(results)) real_res.append(list(cur_label)) ''' ## here means saveModel can multi-ouput sig_def ## results = {} for key in response.outputs: tensor_proto = response.outputs[key] nd_array = tf.contrib.util.make_ndarray(tensor_proto) results[key] = nd_array for key, values in results.items(): print ('in result:', key, values) ''' pred_res = np.array(pred_res).reshape((-1, 1)) real_res = np.array(pred_res).reshape((-1, 1)) res = np.concatenate((real_res, pred_res), axis=1) print('real-label\t pred-label \n', res) return np.sum(np.equal(res[:, 0], res[:, 1])) / res.shape[0]
def get_interactive_infer_results(model, model_in): fetches = [ model.get_data_layer().input_tensors, model.get_output_tensors(), ] feed_dict = model.get_data_layer().create_feed_dict(model_in) # inputs, outputs = sess.run(fetches, feed_dict=feed_dict) # export_path = "/tmp/speech2text/0" # print('Exporting trained model to', export_path) # builder = tf.saved_model.builder.SavedModelBuilder(export_path) # # Define input tensors # audio = tf.saved_model.utils.build_tensor_info( # model.get_data_layer().input_tensors["source_tensors"][0]) # audio_length = tf.saved_model.utils.build_tensor_info( # model.get_data_layer().input_tensors["source_tensors"][1]) # x_id = tf.saved_model.utils.build_tensor_info( # model.get_data_layer().input_tensors["source_ids"][0]) # # Define output tensors # # decoded_sequence = tf.saved_model.utils.build_tensor_info( # # model.get_output_tensors()[0]) # # prediction_signature = ( # # tf.saved_model.signature_def_utils.build_signature_def( # # inputs={'audio': audio, 'audio_length': audio_length, 'x_id': x_id}, # # outputs={'decoded_sequence': decoded_sequence}, # # method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME)) # indices_decoded_sequence = tf.saved_model.utils.build_tensor_info( # model.get_output_tensors()[0].indices) # values_decoded_sequence = tf.saved_model.utils.build_tensor_info( # model.get_output_tensors()[0].values) # dense_shape_decoded_sequence = tf.saved_model.utils.build_tensor_info( # model.get_output_tensors()[0].dense_shape) # prediction_signature = ( # tf.saved_model.signature_def_utils.build_signature_def( # inputs={'audio': audio, 'audio_length': audio_length, 'x_id': x_id}, # outputs={'indices_decoded_sequence': indices_decoded_sequence, # 'values_decoded_sequence': values_decoded_sequence, # 'dense_shape_decoded_sequence': dense_shape_decoded_sequence}, # method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME)) # builder.add_meta_graph_and_variables( # sess, [tf.saved_model.tag_constants.SERVING], # signature_def_map={ # 'predict_output': # prediction_signature, # }, # main_op=tf.tables_initializer(), # strip_default_attrs=True) # builder.save() audio = feed_dict[model.get_data_layer().input_tensors["source_tensors"] [0]] audio_length = feed_dict[ model.get_data_layer().input_tensors["source_tensors"][1]] x_id = feed_dict[model.get_data_layer().input_tensors["source_ids"][0]] print('audio shape: ', audio.shape) print('audio_length shape: ', audio_length.shape) # inputs, outputs = sess.run(fetches, feed_dict=feed_dict) channel = grpc.insecure_channel('0.0.0.0:8500') stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) request = predict_pb2.PredictRequest() request.model_spec.name = 'speech2text' request.model_spec.signature_name = 'predict_output' request.inputs['audio'].CopyFrom( tf.contrib.util.make_tensor_proto(audio, shape=list(audio.shape))) request.inputs['audio_length'].CopyFrom( tf.contrib.util.make_tensor_proto(audio_length, shape=list(audio_length.shape))) request.inputs['x_id'].CopyFrom( tf.contrib.util.make_tensor_proto(x_id, shape=list(x_id.shape))) result_future = stub.Predict.future(request, 5.0) # 5 seconds exception = result_future.exception() if exception: print(exception) else: print('Result returned from rpc') inputs = model.get_data_layer().input_tensors indices_decoded_sequence = tensor_util.MakeNdarray( result_future.result().outputs['indices_decoded_sequence']) values_decoded_sequence = tensor_util.MakeNdarray( result_future.result().outputs['values_decoded_sequence']) dense_shape_decoded_sequence = tensor_util.MakeNdarray( result_future.result().outputs['dense_shape_decoded_sequence']) outputs = tf.SparseTensorValue(indices=indices_decoded_sequence, values=values_decoded_sequence, dense_shape=dense_shape_decoded_sequence) outputs = [outputs] return model.infer(inputs, outputs)
def main(req: func.HttpRequest, context: func.Context) -> func.HttpResponse: _NAME = 'image' event_id = context.invocation_id logging.info( f"Python humanpose function start process.\nID:{event_id}\nBack-end server host: {_HOST}:{_PORT}" ) try: method = req.method url = req.url files = req.files[_NAME] if method != 'POST': logging.warning( f'ID:{event_id},the method was {files.content_type}.refused.') return func.HttpResponse(f'only accept POST method', status_code=400) if files: if files.content_type != 'image/jpeg': logging.warning( f'ID:{event_id},the file type was {files.content_type}.refused.' ) return func.HttpResponse(f'only accept jpeg images', status_code=400) # pre processing # get image_bin form request img_bin = files.read() img = to_pil_image(img_bin) img_cv_copied = cv2.cvtColor(np.asarray(img), cv2.COLOR_RGB2BGR) # w,h = 256,256 img = resize(img) img_np = np.array(img) img_np = img_np.astype(np.float32) # hwc > bchw [1,3,256,256] img_np = transpose(img_np) # print(img_np.shape) request = predict_pb2.PredictRequest() request.model_spec.name = 'object-detection' request.inputs["image"].CopyFrom( make_tensor_proto(img_np, shape=img_np.shape)) # send to infer model by grpc start = time() channel = grpc.insecure_channel("{}:{}".format(_HOST, _PORT)) stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) result = stub.Predict(request, timeout=10.0) res = make_ndarray(result.outputs["detection_out"]) logging.warning(f'OutputType:{type(result)}') # Change a shape of a numpy.ndarray with results ([1, 1, N, 7]) to get another one ([N, 7]), # where N is the number of detected bounding boxes detections = res.reshape(-1, 7) img_copied = np.copy(img_cv_copied) image_h, image_w = img_copied.shape[:2] for i, detection in enumerate(detections): _, class_id, confidence, xmin, ymin, xmax, ymax = detection if confidence > 0.5: xmin = int(xmin * image_w) ymin = int(ymin * image_h) xmax = int(xmax * image_w) ymax = int(ymax * image_h) #logging.info(f'Found: label = {label}, confidence = {confidence:.2f}, ' f'coords = ({xmin}, {ymin}), ({xmax}, {ymax})') # Draw a bounding box on a output image cv2.rectangle(img_copied, (xmin, ymin), (xmax, ymax), (0, 255, 0), 2) timecost = time() - start logging.info(f"Inference complete,Takes{timecost}") # post processing response_image = img_copied imgbytes = cv2.imencode('.jpg', response_image)[1].tobytes() MIMETYPE = 'image/jpeg' return func.HttpResponse(body=imgbytes, status_code=200, mimetype=MIMETYPE, charset='utf-8') else: logging.warning(f'ID:{event_id},Failed to get image,down.') return func.HttpResponse(f'no image files', status_code=400) except grpc.RpcError as e: status_code = e.code() if "DEADLINE_EXCEEDED" in status_code.name: logging.error(e) return func.HttpResponse(f'the grpc request timeout', status_code=408) else: logging.error(f"grpcError:{e}") return func.HttpResponse(f'Failed to get grpcResponse', status_code=500) except Exception as e: logging.error(f"Error:{e}\n\ url:{url}\n\ method:{method}\n") return func.HttpResponse(f'Service Error.check the log.', status_code=500)
def main(): credentials = grpc.ssl_channel_credentials( root_certificates=ROOT_CERT.encode()) channel = grpc.secure_channel( '{}:{}'.format(MODEL_SERVER_HOST, MODEL_SERVER_PORT), credentials) stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) # get the sentences of input sentences = globals.request.form.to_dict() # convert single sentence to feature tokenizer = tokenization.FullTokenizer(vocab_file=VOCAB_FILE_PATH, do_lower_case=True) # Construct the request to tensorflow serving request = predict_pb2.PredictRequest() request.model_spec.name = MODEL_NAME request.model_spec.signature_name = 'serving_default' results = {} for key, sentence in sentences.items(): example = run_classifier.InputExample( guid="test-0", text_a=tokenization.convert_to_unicode(sentence), text_b=None, label=LABELS_LIST[0]) feature = run_classifier.convert_single_example( 0, example, LABELS_LIST, MAX_SEQ_LENGTH, tokenizer) # get the input of model input_ids = np.reshape([feature.input_ids], (1, MAX_SEQ_LENGTH)) input_mask = np.reshape([feature.input_mask], (1, MAX_SEQ_LENGTH)) segment_ids = np.reshape([feature.segment_ids], (MAX_SEQ_LENGTH)) label_ids = [feature.label_id] # package the input into request, Note the format of the input(follow the model) request.inputs['input_ids'].CopyFrom( tf.contrib.util.make_tensor_proto(input_ids, shape=[1, MAX_SEQ_LENGTH], dtype=tf.int32)) request.inputs['input_mask'].CopyFrom( tf.contrib.util.make_tensor_proto(input_mask, shape=[1, MAX_SEQ_LENGTH], dtype=tf.int32)) request.inputs['label_ids'].CopyFrom( tf.contrib.util.make_tensor_proto(label_ids, shape=[1], dtype=tf.int32)) request.inputs['segment_ids'].CopyFrom( tf.contrib.util.make_tensor_proto(segment_ids, shape=[1, MAX_SEQ_LENGTH], dtype=tf.int32)) # do predict result = stub.Predict( request, 100, metadata=metadata_transformer()) # 100 secs timeout # parse the result probabilities_tensor_proto = result.outputs["probabilities"] probabilities = list(probabilities_tensor_proto.float_val) probabilities_np = np.array(probabilities) top3_index_np = probabilities_np.argsort()[-3:][::-1] probabilities_top3 = probabilities_np[top3_index_np] label_top3 = np.array(LABELS_LIST)[top3_index_np] # shape = tf.TensorShape(probabilities_tensor_proto.tensor_shape) # probabilities = np.array(probabilities_tensor_proto.float_val).reshape( # shape.as_list()) result_list = [] for index in range(3): result_list.append({ "label": label_top3[index], "score": str(probabilities_top3[index]) }) results[key] = result_list return Response(json.dumps(results), mimetype='application/json')
def do_inference(hostport, work_dir, concurrency, num_tests): #test_data_set = DataRead('../data/seq.train.data.20191127.part') #test_data_set = DataRead('../data/seq.test.data.20191128.part') test_data_set = DataReadRaw( \ file_input = '../data/cur.day.testdata', \ seq_len = seq_len, \ feat_len = user_len + seq_len*item_len) test_data_set = test_data_set.data_read.values print(getNow(), 'test_set.shape', test_data_set.shape, type(test_data_set)) channel = grpc.insecure_channel(hostport) stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) pred_res = [] real_res = [] pred_label = [] pred_logits = [] pred_logits_ = [] atime = [] atime1 = [] request = predict_pb2.PredictRequest() request.model_spec.name = 'seq_model' ## model name request.model_spec.signature_name = 'predictByRaw' ## signature_def_map ## for index in range(int(num_tests / concurrency)): cur_data = test_data_set[index * concurrency:(index + 1) * concurrency] print('cur_data.shape=', cur_data.shape) cur_label = cur_data[:, 0:seq_len] cur_feature = cur_data[:, seq_len:].reshape( (-1, user_len + seq_len * item_len)) #print ('index=', index, 'cur_feature.shape=', cur_feature.shape, 'cur_feature=', cur_feature) request.inputs['input'].CopyFrom( tf.contrib.util.make_tensor_proto(values=cur_feature, dtype=tf.string, shape=cur_feature.shape)) btime = time.time() response = stub.Predict(request, 5.0) etime = time.time() atime.append(etime - btime) try: results = tf.contrib.util.make_ndarray( response.outputs['predict_prob']) results1 = tf.contrib.util.make_ndarray( response.outputs['predict_label']) results2 = tf.contrib.util.make_ndarray( response.outputs['predict_logits']) #results3 = tf.contrib.util.make_ndarray(response.outputs['predict_logits_']) results3 = tf.contrib.util.make_ndarray( response.outputs['predict_logits_no']) except: continue atime1.append(etime - btime) #print ('index=', index, 'result=', results) pred_res.append(list(results)) real_res.append(list(cur_label)) pred_label.append(list(results1)) pred_logits.append(list(results2)) pred_logits_.append(list(results3)) pred_res = np.array(pred_res).reshape((-1, 14)) real_res = np.array(real_res).reshape((-1, 14)) pred_label = np.array(pred_label).reshape((-1, 14)) pred_logits = np.array(pred_logits).reshape((-1, 14)) pred_logits_ = np.array(pred_logits_).reshape((-1, 14)) count, e_count, p_count, n_count = 0, 0, 0, 0 e_p_count, e_n_count = 0, 0 index_zero_p_count = 0 index_zero_p_pred = 0 index_zero_n_count = 0 index_zero_n_pred = 0 for r in range(pred_res.shape[0]): for z in range(14): count += 1 if real_res[r][z] == pred_label[r][z]: e_count += 1 if real_res[r][z] == 1: p_count += 1 if real_res[r][z] == pred_label[r][z]: e_p_count += 1 if real_res[r][z] == 0: n_count += 1 if real_res[r][z] == pred_label[r][z]: e_n_count += 1 if real_res[r][0] == 1: index_zero_p_count += 1 if pred_label[r][0] == 1: index_zero_p_pred += 1 else: index_zero_n_count += 1 if pred_label[r][0] == 0: index_zero_n_pred += 1 print('real_label', '\t'.join([str(int(x)) for x in real_res[r]])) print('pred_label', '\t'.join([str(int(x)) for x in pred_label[r]])) print('pred_logits', '\t'.join([str(round(x, 3)) for x in pred_logits[r]])) print('pred_prob', '\t'.join([str(round(x, 3)) for x in pred_res[r]])) print('pred_logits_', '\t'.join([str(round(x, 3)) for x in pred_logits_[r]])) print( 'pred_logits_[prob]', '\t'.join([ str(round(1 / (1 + math.exp(-x)), 4)) for x in pred_logits_[r] ])) print('===' * 40) print('equal_count/count=', round(e_count / count * 1.0, 4), e_count, count) print('positive equal_count/count=', round(e_p_count / p_count * 1.0, 4), e_p_count, p_count) print('negative equal_count/count=', round(e_n_count / n_count * 1.0, 4), e_n_count, n_count) print('index-0, pred-1/label-1', round(index_zero_p_pred / index_zero_p_count * 1.0, 4), index_zero_p_pred, index_zero_p_count) print('index-0, pred-0/label-0', round(index_zero_n_pred / index_zero_n_count * 1.0, 4), index_zero_n_pred, index_zero_n_count) print('ava-time[all]:', np.mean(atime)) print('ava-time[success]:', np.mean(atime1))
def img_to_emb_feature(images, channel): # print(img.shape) img = [] for i in range(len(images)): img.append(prewhiten(images[i])) img = np.stack(img) stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) request = predict_pb2.PredictRequest() request.model_spec.name = 'facenet' request.model_spec.signature_name = 'calculate_embeddings' # print('request ', request) request.inputs['images'].CopyFrom( tf.contrib.util.make_tensor_proto(img, dtype=tf.float32)) request.inputs['phase'].CopyFrom(tf.contrib.util.make_tensor_proto(False)) # print('========================') result_tmp = stub.Predict(request, 15.0) # 10 secs timeout # print(result_tmp) result = result_tmp.outputs['embeddings'].float_val # request.model_spec.name = 'facenet' # request.model_spec.signature_name = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY # request.inputs['input'].CopyFrom( # tf.contrib.util.make_tensor_proto(img, shape=[1, img.shape[1], img.shape[2], img.shape[3]])) # result = stub.Predict(request, 10.0) # 10 secs timeout # print("result: ", result) # boxes = np.array(result.outputs['embeddings'].float_val).reshape( # result.outputs['detection_boxes'].tensor_shape.dim[0].size, # result.outputs['detection_boxes'].tensor_shape.dim[1].size, # result.outputs['detection_boxes'].tensor_shape.dim[2].size # ) # # scores = np.array(result.outputs['detection_scores'].float_val) # detection_classes = np.array(result.outputs['detection_classes'].float_val) # # # num_detections = np.array(result.outputs['num_detections'].float_val) # boxes = np.squeeze(boxes) # scores = np.squeeze(scores) # height, width = img.shape[:2] # # pts_box = [] # pts = None # door_img = None # scores_max = 0 # detection_class = None # for i in range(boxes.shape[0]): # if (scores[i] > 0.5) and (scores[i] > scores_max): # scores_max = scores[i] # ymin, xmin, ymax, xmax = boxes[i] # ymin = int(ymin * height) # ymax = int(ymax * height) # xmin = int(xmin * width) # xmax = int(xmax * width) # # pts = np.array([xmin, ymin, xmax, ymax]) # # detection_class = detection_classes[i] # # # channel.close() return result
def transform_view(): from flask import request csv = request.files['file'] csv.save("iBeacon_RSSI_Unlabeled.csv") BLE_RSSI_UL = pd.read_csv("iBeacon_RSSI_Unlabeled.csv", encoding='utf8') # Unlabeled dataset COLUMNS = list(BLE_RSSI_UL.columns) FEATURES = COLUMNS[2:] LABEL = [COLUMNS[0]] # Data Preprocesssing df_predict = BLE_RSSI_UL # Unlabeled dataset df_predict = df_predict.drop(['date', 'location'], axis=1) df_predict_original = df_predict.copy() # df_predict[FEATURES] = (df_predict[FEATURES] - df_predict[FEATURES].mean()) / df_predict[FEATURES].std() df_predict[FEATURES] = (df_predict[FEATURES]) / (-200) max_rows = 2800 dataframes = [] dataframes_original = [] while len(df_predict) > max_rows: top_original = df_predict_original[:max_rows] top = df_predict[:max_rows] dataframes.append(top) dataframes_original.append(top_original) df_predict = df_predict[max_rows:] df_predict_original = df_predict_original[max_rows:] else: dataframes.append(df_predict) dataframes_original.append(df_predict_original) # server = str(TF_MODEL_SERVER_HOST)+":"+str(TF_MODEL_SERVER_PORT) final_df = pd.DataFrame(columns=[ 'b3001', 'b3002', 'b3003', 'b3004', 'b3005', 'b3006', 'b3007', 'b3008', 'b3009', 'b3010', 'b3011', 'b3012', 'b3013', 'Location', 'Probability' ]) for i, j in zip(dataframes, dataframes_original): j.index = pd.RangeIndex(len(i.index)) j.index = range(len(i.index)) examples = [] for index, row in i.iterrows(): example = tf.train.Example() for col, value in row.iteritems(): example.features.feature[col].float_list.value.append(value) examples.append(example) # channel = grpc.insecure_channel(server) stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) request = classification_pb2.ClassificationRequest() request.model_spec.name = 'Model_Blerssi' request.model_spec.signature_name = 'serving_default' request.input.example_list.examples.extend(examples) response = stub.Classify(request, 10.0) outputs = j.copy() for index, row in outputs.iterrows(): max_class = max(response.result.classifications[index].classes, key=lambda c: c.score) outputs.loc[index, 'Location'] = get_key(int(max_class.label)) outputs.loc[index, 'Probability'] = max_class.score print(outputs) final_df = final_df.append(outputs, ignore_index=True) os.remove("iBeacon_RSSI_Unlabeled.csv") return render_template('view.html', tables=[final_df.to_html()], titles=['na'])
def create_channel_for_batching_server_auto(): channel = grpc.insecure_channel('localhost:9005') stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) return stub
def create_channel_for_model_ver_pol_server(): channel = grpc.insecure_channel('localhost:9006') stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) return stub
def predict(): # MODEL PARAMS logging.basicConfig( level=logging.DEBUG, format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", ) logger = logging.getLogger(__name__) logger.info("Sending request to tfserving model") # Create gRPC client and request t = time.time() channel = grpc.insecure_channel("bert-toxic:8500") logger.debug("Establishing insecure channel took: {}".format(time.time() - t)) t = time.time() stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) logger.debug("Creating stub took: {}".format(time.time() - t)) # Parse Description tokenizer = tokenization.FullTokenizer(vocab_file="asset/vocab.txt", do_lower_case=True) processor = MultiLabelTextProcessor() label_list = [0, 0, 0, 0, 0, 0] t = time.time() content = request.get_json() logger.debug("Parsing Incoming JSON took: {}".format(time.time() - t)) max_seq_length = content["max_seq_length"] request_id = str(random.randint(1, 9223372036854775807)) t = time.time() input_strings = content["description"] model_inputs = [] for string in input_strings: inputExample = processor.serving_create_example([request_id, string], "test") logger.debug("Create Input Example Took: {}".format(time.time() - t)) t = time.time() feature = convert_single_example(0, inputExample, label_list, max_seq_length, tokenizer) logger.debug("Feature Creation Took: {}".format(time.time() - t)) t = time.time() features = collections.OrderedDict() features["input_ids"] = create_int_feature(feature.input_ids) features["input_mask"] = create_int_feature(feature.input_mask) features["segment_ids"] = create_int_feature(feature.segment_ids) features["is_real_example"] = create_int_feature( [int(feature.is_real_example)]) if isinstance(feature.label_id, list): label_ids = feature.label_id else: label_ids = [feature.label_id] features["label_ids"] = create_int_feature(label_ids) tf_example = tf.train.Example(features=tf.train.Features( feature=features)) model_input = tf_example.SerializeToString() model_inputs.append(model_input) logger.debug("Serialize Features Took: {}".format(time.time() - t)) t = time.time() # Send request # See prediction_service.proto for gRPC request/response details. model_request = predict_pb2.PredictRequest() model_request.model_spec.name = "bert" model_request.model_spec.signature_name = "serving_default" dims = [tensor_shape_pb2.TensorShapeProto.Dim(size=len(model_inputs))] tensor_shape_proto = tensor_shape_pb2.TensorShapeProto(dim=dims) tensor_proto = tensor_pb2.TensorProto( dtype=types_pb2.DT_STRING, tensor_shape=tensor_shape_proto, string_val=[tweet for tweet in model_inputs], ) logger.debug("Format Tensors Took: {}".format(time.time() - t)) t = time.time() model_request.inputs["examples"].CopyFrom(tensor_proto) logger.debug("Create Model Request Inputs: {}".format(time.time() - t)) t = time.time() result = stub.Predict(model_request, 10.0) # 10 secs timeout logger.debug("stub predict took: {}".format(time.time() - t)) t = time.time() predict_response_dict = predict_response_to_dict(result) logger.debug("predict_response_dict took: {}".format(time.time() - t)) t = time.time() keys = [k for k in predict_response_dict] logger.debug("Receive and Iterate Took: {}".format(time.time() - t)) t = time.time() def format_dict(response): f_dict = { "toxic": round(response[0], 4), "severe_toxic": round(response[1], 4), "obscene": round(response[2], 4), "threat": round(response[3], 4), "insult": round(response[4], 4), "identity_hate": round(response[5], 4), } return f_dict response_list = [ format_dict(response) for response in predict_response_dict["probabilities"] ] label_dict = {"results": response_list} logger.debug("Create Label Dict Took: {}".format(time.time() - t)) return jsonify(label_dict)
def create_channel_for_port_mapping_server(): channel = grpc.insecure_channel('localhost:9002') stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) return stub
def create_grpc_stub(host, port=8500): hostport = '{}:{}'.format(host, port) channel = grpc.insecure_channel(hostport) stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) return stub
def main(_): s = open('mscoco_complete_label_map.pbtxt', 'r').read() mymap = labelmap.StringIntLabelMap() global _label_map _label_map = text_format.Parse(s, mymap) channel = grpc.insecure_channel(FLAGS.server) stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) durationSum = 0.0 run_num = 1 batch_size = 1 image, org = decode_image_opencv(FLAGS.image) _draw = org.copy() image = image.astype(np.uint8) inputs = image for i in range(batch_size - 1): inputs = np.append(inputs, image, axis=0) model_name = "ssd_inception_v2_coco" # model_name = "ssd_mobilenet" # model_name = "ssd_resnet50" # model_name = "faster_rcnn_resnet50" request = predict_pb2.PredictRequest() request.model_spec.name = model_name request.model_spec.signature_name = 'serving_default' request.inputs['inputs'].CopyFrom( tf.contrib.util.make_tensor_proto(inputs, shape=inputs.shape)) for i in range(run_num): start = time.time() result = stub.Predict(request, 10.0) end = time.time() duration = end - start durationSum += duration print("duration = %f" % duration) # print(result) boxes = result.outputs['detection_boxes'] scores = result.outputs['detection_scores'] labels = result.outputs['detection_classes'] num_detections = result.outputs['num_detections'] boxes = tf.make_ndarray(boxes) scores = tf.make_ndarray(scores) labels = tf.make_ndarray(labels) num_detections = tf.make_ndarray(num_detections) print("boxes output", (boxes).shape) print("scores output", (scores).shape) print("labels output", (labels).shape) print('num_detections', num_detections[0]) # visualize detections hints from # # https://github.com/tensorflow/models/blob/master/research/object_detection/object_detection_tutorial.ipynb for box, score, label in zip(boxes[0], scores[0], labels[0]): # scores are sorted so we can break if score < 0.5: break #dim = image.shape[0:2] dim = _draw.shape #print("Label-raw",labels_to_names[label-1]," at ",box," Score ",score) box = box_normal_to_pixel(box, dim) b = box.astype(int) class_label = get_label(int(label)) print("Label", class_label, " at ", b, " Score ", score) # draw the image and write out cv2.rectangle(_draw, (b[0], b[1]), (b[2], b[3]), (0, 0, 255), 1) cv2.putText(_draw,class_label + "-"+str(round(score,2)), (b[0]+2,b[1]+8),\ cv2.FONT_HERSHEY_SIMPLEX, .45, (0,0,255)) cv2.imshow("test", _draw) cv2.waitKey(0) print("average duration for batch size of %d = %f" % (batch_size, durationSum / run_num))
def benchmark(batch_size=1, num_iteration=20, warm_up_iteration=10): channel = grpc.insecure_channel(SERVER_URL) stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) request = predict_pb2.PredictRequest() request.model_spec.name = 'wide-deep-large-ds-fp32-training' request.model_spec.signature_name = 'predict' i = 0 total_time = 0 res_dataset = generate_input_fn(DATA_FILE, batch_size=batch_size, num_epochs=1) #res_dataset = input_fn(DATA_FILE, 1, False, batch_size) CONTINUOUS_COLUMNS = ["I" + str(i) for i in range(1, 14)] # 1-13 inclusive CATEGORICAL_COLUMNS = ["C" + str(i) for i in range(1, 27)] # 1-26 inclusive LABEL_COLUMN = ["clicked"] TRAIN_DATA_COLUMNS = LABEL_COLUMN + CONTINUOUS_COLUMNS + CATEGORICAL_COLUMNS FEATURE_COLUMNS = CONTINUOUS_COLUMNS + CATEGORICAL_COLUMNS iterator = tf.compat.v1.data.make_one_shot_iterator(res_dataset) next_element = iterator.get_next() with tf.compat.v1.Session() as sess: sess.run(tf.compat.v1.global_variables_initializer()) try: for _ in range(num_iteration): correct_predictions = 0 i += 1 input_data = sess.run(next_element) for c in FEATURE_COLUMNS: request.inputs[c].CopyFrom( tf.make_tensor_proto(input_data[0][c], shape=[1 * batch_size])) # request.inputs['numeric'].CopyFrom( # tf.make_tensor_proto(input_data[0], # shape=[1 * batch_size, 13])) # request.inputs['categorical'].CopyFrom( # tf.make_tensor_proto(input_data[1], # shape=[26 * batch_size, 2])) start_time = time.time() result = stub.Predict(request) # print("-" * 20 + " result " + "-" * 20) # print(result.outputs['probabilities']) # print("-" * 20 + " result " + "-" * 20) time_consume = time.time() - start_time if i > warm_up_iteration: total_time += time_consume for j in range(batch_size): if input_data[1][j] == np.argmax( result.outputs['probabilities'].float_val[ j * 2:(j * 2) + 2]): correct_predictions += 1 print('Iteration %d: %.3f sec (accuracy: %.2f%%)' % ( i, time_consume, 100.0 * correct_predictions / batch_size)) except tf.errors.OutOfRangeError: pass time_average = total_time / (num_iteration - warm_up_iteration) print('Average time: %.3f sec' % (time_average)) print('Batch size = %d' % batch_size) if batch_size == 1: print('Latency: %.3f ms' % (time_average * 1000)) print('Throughput: %.3f examples/sec' % (batch_size / time_average))
def __init__(self): channel = grpc.insecure_channel("localhost:8500") self.stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) pass
def create_channel_for_update_flow_specific(): channel = grpc.insecure_channel('localhost:9008') stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) return stub
def _create_stub(server): channel = grpc.insecure_channel(server) return prediction_service_pb2_grpc.PredictionServiceStub(channel)
def __init__(self, ip_port): channel = grpc.insecure_channel(ip_port) self.stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
async def __connection__(self, task_idx, loop_num): request_signatures = self.request_signatures[task_idx] response_list = [] # create channel if self.certificate == None: async with grpc.aio.insecure_channel(self.url) as channel: stub = prediction_service_pb2_grpc.PredictionServiceStub( channel) if loop_num != 0: format_string = 'query: {} channel, task {}, batch {}, loop_idx {}, latency(ms) {:.1f}, tps: {:.1f}' for loop_idx in range(loop_num): start_time = time.time() response = await stub.Predict(request_signatures) stop_time = time.time() latency = stop_time - start_time tps = self.batch_size / latency response_list.append([response, latency]) print( format_string.format('insecure', task_idx, self.batch_size, loop_idx, 1000 * latency, tps)) else: format_string = 'query: {} channel, task {}, batch {}, latency(ms) {:.1f}, tps: {:.1f}' while True: start_time = time.time() response = await stub.Predict(request_signatures) stop_time = time.time() latency = stop_time - start_time tps = self.batch_size / latency print( format_string.format('insecure', task_idx, self.batch_size, 1000 * latency, tps)) else: creds = grpc.ssl_channel_credentials( root_certificates=open(self.certificate, 'rb').read()) async with grpc.aio.secure_channel(self.url, creds) as channel: stub = prediction_service_pb2_grpc.PredictionServiceStub( channel) if loop_num != 0: format_string = 'query: {} channel, task {}, batch {}, loop_idx {}, latency(ms) {:.1f}, tps: {:.1f}' for loop_idx in range(loop_num): start_time = time.time() response = await stub.Predict(request_signatures) stop_time = time.time() latency = stop_time - start_time tps = self.batch_size / latency response_list.append([response, latency]) print( format_string.format('secure', task_idx, self.batch_size, loop_idx, 1000 * latency, tps)) else: format_string = 'query: {} channel, task {}, batch {}, latency(ms) {:.1f}, tps: {:.1f}' while True: start_time = time.time() response = await stub.Predict(request_signatures) stop_time = time.time() latency = stop_time - start_time tps = self.batch_size / latency try: proto_msg_to_dict(response) except Exception as e: print('Error response:', e) print( format_string.format('secure', task_idx, self.batch_size, 1000 * latency, tps)) return response_list
def do_inference(): """Tests PredictionService with concurrent requests. Raises: IOError: An error occurred processing test data set. """ if FLAGS.mode == 'dna': CONF = DNA_CONF() elif FLAGS.mode == 'rna': CONF = RNA_CONF() else: raise ValueError("Mode has to be either rna or dna.") make_dirs(FLAGS.output) FLAGS.segment_len = CONF.SEGMENT_LEN FLAGS.jump = CONF.JUMP FLAGS.start = CONF.START pbars = multi_pbars(["Request Submit:", "Request finished"]) channel = grpc.insecure_channel(FLAGS.server) stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) request = predict_pb2.PredictRequest() request.model_spec.name = 'chiron' # request.model_spec.signature_name = 'predicted_sequences' request.model_spec.signature_name = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY collector = _Result_Collection(concurrency=FLAGS.concurrency) file_list = gen_file_list(FLAGS.input) batch_iterator = data_iterator(file_list) def submit_fn(): for batch_x, seq_len, i, f, N, reads_n in batch_iterator: seq_len = np.reshape(seq_len, (seq_len.shape[0], 1)) # combined_input = np.concatenate((batch_x,seq_len),axis = 1).astype(np.float32) # request.inputs['combined_inputs'].CopyFrom( # tf.contrib.util.make_tensor_proto(combined_input, shape=[FLAGS.batch_size, CONF.SEGMENT_LEN+1])) request.inputs['x'].CopyFrom( tf.contrib.util.make_tensor_proto( batch_x, shape=[FLAGS.batch_size, CONF.SEGMENT_LEN])) request.inputs['seq_len'].CopyFrom( tf.contrib.util.make_tensor_proto(seq_len, shape=[FLAGS.batch_size])) collector.throttle() result_future = stub.Predict.future(request, 100.0) # 5 seconds result_future.add_done_callback( _post_process(collector, i, f, N, reads_n)) pbars.update(0, total=reads_n, progress=(i + 1) * FLAGS.batch_size) pbars.update_bar() submiter = threading.Thread(target=submit_fn, args=()) submiter.setDaemon(True) submiter.start() pbars.update(1, total=len(file_list)) pbars.update_bar() while not collector.all_done(): if len(collector._done) > 0: qs_string = None f_p = collector._done[0] reads, probs = collector.pop_out(f_p) bpreads = [index2base(read) for read in reads] consensus, qs_consensus = simple_assembly_qs(bpreads, probs) qs_string = qs(consensus, qs_consensus) c_bpread = index2base(np.argmax(consensus, axis=0)) file_pre = os.path.basename(os.path.splitext(f_p)[0]) write_output(bpreads, c_bpread, [np.NaN] * 4, file_pre, concise=FLAGS.concise, suffix=FLAGS.extension, q_score=qs_string, global_setting=FLAGS) pbars.update(1, progress=pbars.progress[1] + 1) pbars.update_bar()
def make_grpc_api_call(context_answer, batch_size, server_host, server_port, vocab_file, max_seq_length=512, do_lower_case=False, server_name="bert_QG", timeout=60.0): input_examples = [] for tup in context_answer: context, answer = tup input_example = create_example(context, answer) input_examples.append(input_example) tokenizer = tokenization.FullTokenizer(vocab_file=vocab_file, do_lower_case=do_lower_case) serialized_tf_examples = [] for input_example in input_examples: serialized_tf_example = convert_example_to_feature( input_example, max_seq_length, tokenizer) serialized_tf_examples.append(serialized_tf_example) if batch_size > len(serialized_tf_examples): batch_size = len(serialized_tf_examples) num_batches = int(len(serialized_tf_examples) / batch_size) if num_batches % batch_size != 0: num_batches = num_batches + 1 # all examples pred_questions = [] for i in range(num_batches): # get the batch of serialized tf example inp_batch = serialized_tf_examples[i * batch_size:(i + 1) * batch_size] if len(inp_batch) == 0: continue # update the server host:port for making grpc api call # create the RPC stub server_grpc = server_host + ':' + server_port channel = grpc.insecure_channel(server_grpc) stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) # create the request object and set the name and signature_name params request = predict_pb2.PredictRequest() request.model_spec.name = server_name request.model_spec.signature_name = 'serving_default' # fill in the request object with the necessary data request.inputs['examples'].CopyFrom( tf.make_tensor_proto(inp_batch, dtype=tf.string, shape=[len(inp_batch)])) # sync requests with 30 sec wait before termination # result_future = stub.Predict(request, 30.) # For async requests result_future = stub.Predict.future(request, timeout) result_future = result_future.result() # Get shape of batch and categories in prediction NUM_PREDICTIONS = \ result_future.outputs['pred_ids'].tensor_shape.dim[0].size NUM_TOKENS = \ result_future.outputs['pred_ids'].tensor_shape.dim[1].size batch_preds = np.reshape(result_future.outputs['pred_ids'].int32_val, (int(NUM_PREDICTIONS), int(NUM_TOKENS))) for (i, pred_ids) in enumerate(batch_preds): pred_tokens = tokenizer.convert_ids_to_tokens(pred_ids) output = [] for tok in pred_tokens: if tok == "[SEP]": break output.append(tok) question_text = " ".join(output) pred_questions.append(question_text) return pred_questions
def create_grpc_stub(self, grpc_channel): return prediction_service_pb2_grpc.PredictionServiceStub(grpc_channel)
def start(args): api = None try: ctx = Context(s3_path=args.context, cache_dir=args.cache_dir, workload_id=args.workload_id) api = ctx.apis_id_map[args.api] local_cache["api"] = api local_cache["ctx"] = ctx if api.get("request_handler") is not None: local_cache["request_handler"] = ctx.get_request_handler_impl( api["name"], args.project_dir) request_handler = local_cache.get("request_handler") if request_handler is not None and util.has_function( request_handler, "pre_inference"): logger.info( "using pre_inference request handler provided in {}".format( api["request_handler"])) else: logger.info("pre_inference request handler not found") if request_handler is not None and util.has_function( request_handler, "post_inference"): logger.info( "using post_inference request handler provided in {}".format( api["request_handler"])) else: logger.info("post_inference request handler not found") except Exception as e: logger.exception("failed to start api") sys.exit(1) try: validate_model_dir(args.model_dir) except Exception as e: logger.exception("failed to validate model") sys.exit(1) if api.get("tracker") is not None and api["tracker"].get( "model_type") == "classification": try: local_cache["class_set"] = api_utils.get_classes(ctx, api["name"]) except Exception as e: logger.warn("an error occurred while attempting to load classes", exc_info=True) channel = grpc.insecure_channel("localhost:" + str(args.tf_serve_port)) local_cache["stub"] = prediction_service_pb2_grpc.PredictionServiceStub( channel) # wait a bit for tf serving to start before querying metadata limit = 60 for i in range(limit): try: local_cache["model_metadata"] = run_get_model_metadata() break except Exception as e: if i > 6: logger.warn( "unable to read model metadata - model is still loading. Retrying..." ) if i == limit - 1: logger.exception("retry limit exceeded") sys.exit(1) time.sleep(5) signature_key, parsed_signature = extract_signature( local_cache["model_metadata"]["signatureDef"], api["tf_signature_key"]) local_cache["signature_key"] = signature_key local_cache["parsed_signature"] = parsed_signature logger.info("model_signature: {}".format(local_cache["parsed_signature"])) serve(app, listen="*:{}".format(args.port))
def main(req: func.HttpRequest, context: func.Context) -> func.HttpResponse: _NAME = 'image' event_id = context.invocation_id logging.info( f"Python yolo-v3 function start process.\nID:{event_id}\nBack-end server host: {_HOST}:{_PORT}") try: method = req.method url = req.url files = req.files[_NAME] if method != 'POST': logging.warning( f'ID:{event_id},the method was {files.content_type}.refused.') return func.HttpResponse(f'only accept POST method', status_code=400) if files: if files.content_type != 'image/jpeg': logging.warning( f'ID:{event_id},the file type was {files.content_type}.refused.') return func.HttpResponse(f'only accept jpeg images', status_code=400) img_bin = files.read() img = prep.to_pil_image(img_bin) img=prep.rotate_image(img) frame = cv2.cvtColor(np.asarray(img), cv2.COLOR_RGB2BGR) # resize image to [416, 416] img = prep.resize(img, w=416, h=416) # img = prep.resize(img, w=608, h=608) img_np = np.array(img) img_np = img_np.astype(np.float32) # hwc > bchw [1,3,416, 416] img_np = prep.transpose(img_np) # semantic segmentation request = predict_pb2.PredictRequest() request.model_spec.name = 'yolo-v3' # request.model_spec.name = 'yolo-v4' request.inputs["input_1"].CopyFrom(make_tensor_proto(img_np)) # request.inputs["image_input"].CopyFrom(make_tensor_proto(img_np)) # send to infer model by grpc start = time() channel = grpc.insecure_channel("{}:{}".format(_HOST, _PORT)) stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) channel = grpc.insecure_channel("{}:{}".format(_HOST, _PORT)) result = stub.Predict(request, timeout=10.0) # logging.warning(f'Output:{result}') logging.warning(f'OutputType:{type(result)}') # print(result.outputs) output1=make_ndarray(result.outputs['conv2d_58/Conv2D/YoloRegion']) output2=make_ndarray(result.outputs['conv2d_66/Conv2D/YoloRegion']) output3=make_ndarray(result.outputs['conv2d_74/Conv2D/YoloRegion']) outputs=[output1, output2, output3] #----------------------------------------------------------- # output image which objects are surrounded with rectangles. # Their labels are shown on them. frame=postp.object_detection(frame, img_np, outputs) #----------------------------------------------------------- timecost = time()-start logging.info(f"Inference complete,Takes{timecost}") imgbytes = cv2.imencode('.jpg', frame)[1].tobytes() # imgbytes = prep.encode(image) MIMETYPE = 'image/jpeg' return func.HttpResponse(body=imgbytes, status_code=200, mimetype=MIMETYPE, charset='utf-8') else: logging.warning(f'ID:{event_id},Failed to get image,down.') return func.HttpResponse(f'no image files', status_code=400) except grpc.RpcError as e: status_code = e.code() if "DEADLINE_EXCEEDED" in status_code.name: logging.error(e) return func.HttpResponse(f'the grpc request timeout', status_code=408) else: logging.error(f"grpcError:{e}") return func.HttpResponse(f'Failed to get grpcResponse', status_code=500) except Exception as e: logging.error(f"Error:{e}\n\ url:{url}\n\ method:{method}\n") return func.HttpResponse(f'Service Error.check the log.', status_code=500)
lemmatizer = WordNetLemmatizer() stemmer = PorterStemmer() stop = set(stopwords.words('english')) # Map data into vocabulary # RNN text_path_rnn = os.path.join(FLAGS.checkpoint_dir_rnn, "..", "text_vocab") #text_path = os.path.join(FLAGS.checkpoint_dir, "..", "text_vocab") text_vocab_processor_rnn = tf.contrib.learn.preprocessing.VocabularyProcessor.restore( text_path_rnn) channel_rnn = grpc.insecure_channel('0.0.0.0:8500') stub_rnn = prediction_service_pb2_grpc.PredictionServiceStub(channel_rnn) request_rnn = predict_pb2.PredictRequest() request_rnn.model_spec.name = 'lstm' # CNN text_path_cnn = os.path.join(FLAGS.checkpoint_dir_cnn, "..", "vocab") #text_path = os.path.join(FLAGS.checkpoint_dir, "..", "text_vocab") text_vocab_processor_cnn = tf.contrib.learn.preprocessing.VocabularyProcessor.restore( text_path_cnn) channel_cnn = grpc.insecure_channel('0.0.0.0:8501') stub_cnn = prediction_service_pb2_grpc.PredictionServiceStub(channel_cnn) request_cnn = predict_pb2.PredictRequest()
def __init__(self): channel = grpc.insecure_channel(FLAGS.server) self.predict_stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) self.model_name = 'segment' self.signature_name = 'predict_segment' self.timeout = 3
import grpc from tensorflow_serving.apis import prediction_service_pb2_grpc from tensorflow_serving.apis import predict_pb2 import sys sys.path.append( '/home/yitao/Documents/fun-project/tensorflow-related/video-captioning-serving/' ) from modules_video_cap.video_cap_alexnet_d2 import CapAlexnet alexnet = CapAlexnet() alexnet.Setup() ichannel = grpc.insecure_channel("localhost:8500") istub = prediction_service_pb2_grpc.PredictionServiceStub(ichannel) video_path = "/home/yitao/Documents/fun-project/tensorflow-related/video-captioning-serving/inputs/vid264.mp4" reader = cv2.VideoCapture(video_path) frame_id = 1 batch_size = 16 while (frame_id < 256): start = time.time() data_array = [] for i in range(batch_size): _, image = reader.read()
def do_inference(server, batch_size, num_tests, img_path): channel = grpc.insecure_channel(server) stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) request = predict_pb2.PredictRequest() request.model_spec.name = 'retinanet_od' request.model_spec.signature_name = 'serving_default' print("Image path", img_path) #post process the image #image,org= decode_image_opencv(img_path) image, org = decode_image_retinanet(img_path) #image,org = decode_image_tf_reader(img_path) global _draw _draw = org.copy() print("in image shape", image.shape) #('in image shape', (480, 640, 3)) #input = np.expand_dims(image, axis=0) # do this once as tf. make_tensor proto is slow on first use #request.inputs['input_image'].CopyFrom(tf.contrib.util.make_tensor_proto \ # (input, shape=[1, 800, 1067, 3])) # (input, shape=[1, 500, 567, 3])) global _start _start = time.time() global _response_awaiting _response_awaiting = True for i in range(num_tests): #print("Going to send the request") # batching # If using anything other than decode_opencv uncomment line below input = np.expand_dims(image, axis=0) #('Input shape=', (1, 480, 640, 3)) #input = image # comment this if using anything other than retinanet print("Input shape=", input.shape) inputs = input for _ in range(batch_size - 1): inputs = np.append(inputs, input, axis=0) print("in tf shape", inputs.shape) request.inputs['input_image'].CopyFrom( tf.contrib.util.make_tensor_proto(inputs, shape=inputs.shape)) # call back way - this is faster result_future = stub.Predict.future(request, 60.25) # Intial takes time result_future.add_done_callback(_callback) # request reponse way - this is slower # result = stub.Predict(request, 10.25) # seconds # parse_result(result) _response_awaiting = True #print("Send the request") # End for loop while (_response_awaiting): time.sleep(.000010) print("Response Received Exiting") ################################################################################ # Helper functions for image preoricessing # ################################################################################ # Going to read the image via TF helper img_raw = tf.read_file(img_path) start = timer() img_tensor = tf.image.decode_jpeg( img_raw, channels=0, dct_method="INTEGER_FAST") #not much effect here decode time 30ms print("img_tensor.shape=", img_tensor.shape)
### Using the gRPC API from tensorflow_serving.apis.predict_pb2 import PredictRequest request = PredictRequest() request.model_spec.name = model_name request.model_spec.signature_name = "serving_default" input_name = model.input_names[0] request.inputs[input_name].CopyFrom(tf.make_tensor_proto(X_new)) import grpc from tensorflow_serving.apis import prediction_service_pb2_grpc channel = grpc.insecure_channel('localhost:8500') predict_service = prediction_service_pb2_grpc.PredictionServiceStub(channel) response = predict_service.Predict(request, timeout=10.0) response Convert the response to a tensor: output_name = model.output_names[0] outputs_proto = response.outputs[output_name] y_proba = tf.make_ndarray(outputs_proto) y_proba.round(2) Or to a NumPy array if your client does not include the TensorFlow library: output_name = model.output_names[0] outputs_proto = response.outputs[output_name]
def predict(filename): channel = grpc.insecure_channel("localhost:8500") stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) request = predict_pb2.PredictRequest() # model_name request.model_spec.name = "cropper_model" # signature name, default is 'serving_default' request.model_spec.signature_name = "serving_default" start = time.time() """ ========================================= ===== Crop and align id card image ========================================= """ filepath = app.config["IMAGE_UPLOADS"] + "/" + filename # preprocess image img, original_image, original_width, original_height = preprocess_image( filepath, Cropper.TARGET_SIZE) if img.ndim == 3: img = np.expand_dims(img, axis=0) # request to cropper model request.inputs["input_1"].CopyFrom( tf.make_tensor_proto(img, dtype=np.float32, shape=img.shape)) try: result = stub.Predict(request, 10.0) result = result.outputs["tf_op_layer_concat_14"].float_val result = np.array(result).reshape((-1, 9)) except Exception as e: print(e) cropper = Cropper() cropper.set_best_bboxes(result, original_width=original_width, original_height=original_height, iou_threshold=0.5) # respone to client if image is invalid if not cropper.respone_client(threshold_idcard=0.8): return render_template('upload_image_again.html') cropper.set_image(original_image=original_image) # output of cropper part aligned_image = getattr(cropper, "image_output") cv2.imwrite('app/static/aligned_images/' + filename, aligned_image) aligned_image = cv2.cvtColor(aligned_image, cv2.COLOR_BGR2RGB) """ =========================================== ==== Detect informations in aligned image =========================================== """ # preprocess aligned image original_height, original_width, _ = aligned_image.shape img = cv2.resize(aligned_image, Detector.TARGET_SIZE) img = np.float32(img / 255.) # model_name request.model_spec.name = "detector_model" # signature name, default is 'serving_default' request.model_spec.signature_name = "serving_default" if img.ndim == 3: img = np.expand_dims(img, axis=0) # new request to detector model request.inputs["input_1"].CopyFrom( tf.make_tensor_proto(img, dtype=np.float32, shape=img.shape)) try: result = stub.Predict(request, 10.0) result = result.outputs["tf_op_layer_concat_14"].float_val result = np.array(result).reshape((-1, 13)) except Exception as e: print(e) detector = Detector() detector.set_best_bboxes(result, original_width=original_width, original_height=original_height, iou_threshold=0.5) detector.set_info_images(original_image=aligned_image) # output of detector part info_images = getattr(detector, "info_images") """ ===================================== ==== Reader infors from infors image ===================================== """ keys = list(info_images.keys()) keys.remove("thoi_han") keys.remove("chan_dung") infors = dict() # init default value of quoc_tich, dan_toc infors['quoc_tich'] = "" infors['dan_toc'] = "" if "quoc_tich" in keys: infors['quoc_tich'] = ["Việt Nam"] keys.remove("quoc_tich") if "sex" in keys: info_image = info_images["sex"] infors["sex"] = list() for i in range(len(info_image)): img = info_image[i]['image'] s = reader.predict(img) if "Na" in s: infors["sex"].append("Nam") else: infors["sex"].append("Nữ") keys.remove("sex") if "dan_toc" in keys: info_image = info_images["dan_toc"] infors["dan_toc"] = list() for i in range(len(info_image)): img = info_image[i]['image'] s = reader.predict(img) s = s.split(" ")[-1] infors["dan_toc"].append(s) keys.remove("dan_toc") for key in keys: infors[key] = list() info_image = info_images[key] for i in range(len(info_image)): img = info_image[i]['image'] s = reader.predict(img) infors[key].append(s) que_quan_0 = infors['que_quan'][0] que_quan_1 = '' noi_thuong_tru_0 = infors['noi_thuong_tru'][0] noi_thuong_tru_1 = '' if len(infors['que_quan']) == 2: que_quan_1 = infors['que_quan'][1] if len(infors['noi_thuong_tru']) == 2: noi_thuong_tru_1 = infors['noi_thuong_tru'][1] print("total_time:{}".format(time.time() - start)) return render_template('predict.html', id=infors['id'][0].replace(" ", ""), full_name=infors['full_name'][0], date_of_birth=infors['date_of_birth'][0], sex=infors['sex'][0], quoc_tich=infors['quoc_tich'], dan_toc=infors['dan_toc'], que_quan_0=que_quan_0, que_quan_1=que_quan_1, noi_thuong_tru_0=noi_thuong_tru_0, noi_thuong_tru_1=noi_thuong_tru_1, filename=str(filename))