def main():
begin = time.time()
features = prcoess_data()
server = '0.0.0.0:9000'
channel = grpc.insecure_channel(server)
stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
request = predict_pb2.PredictRequest()
request.model_spec.name = 'test'
request.model_spec.signature_name = 'serving_default'
# tensor_input_ids = construct_tensor([1, 64], features.input_ids)
# tensor_input_mask = construct_tensor([1, 64], features.input_mask)
# tensor_label_ids = construct_tensor([1, 1], [0])
# tensor_segment_ids = construct_tensor([1, 64], features.segment_ids)
#
# request.inputs['input_ids'].CopyFrom(tensor_input_ids)
# request.inputs['input_mask'].CopyFrom(tensor_input_mask)
# request.inputs['label_ids'].CopyFrom(tensor_label_ids)
# request.inputs['segment_ids'].CopyFrom(tensor_segment_ids)
request.inputs['input_ids'].CopyFrom(
contrib_util.make_tensor_proto(features.input_ids, shape=[1, 64]))
request.inputs['input_mask'].CopyFrom(
contrib_util.make_tensor_proto(features.input_mask, shape=[1, 64]))
request.inputs['label_ids'].CopyFrom(
contrib_util.make_tensor_proto([0], shape=[1, 1]))
request.inputs['segment_ids'].CopyFrom(
contrib_util.make_tensor_proto(features.segment_ids, shape=[1, 64]))
result = stub.Predict(request, 10.0) # 10 secs timeout
end = time.time() - begin
output = np.array(result.outputs['probabilities'].float_val)
print('time {}'.format(end))
print(output)
def process(self, abs_img_dir, image, input_name, model_name, other_k,
signature_name, stub, scale):
request = predict_pb2.PredictRequest()
# 端口里面的名字什么的，设置的第一级为test，第二级为predict_images
request.model_spec.name = model_name
request.model_spec.signature_name = signature_name
# protobuf 序列化并发送请求和接受结果
request.inputs[input_name].CopyFrom(
util.make_tensor_proto(image, dtype=tf.string))
if len(other_k) != 0:
for key in list(other_k.keys()):
name, _type = key.split(':')
if _type == 'int':
this_type = tf.int32
elif _type == 'float':
this_type = tf.float32
request.inputs[name].CopyFrom(
util.make_tensor_proto(other_k[key], dtype=this_type))
result = stub.Predict(request, 90.0) # 时限
# 处理返回的信息
results = {
'result': result,
'abs_img_dir': abs_img_dir,
'scale': scale
}
# os.popen("chmod 777 " + os.path.join(self.dir_path, 'media', abs_img_dir)).readlines()
return results
def predict(host, port, hostname, model, signature_name, input_path):
# If hostname not set, we assume the host is a valid knative dns.
if hostname:
host_option = ((
'grpc.ssl_target_name_override',
hostname,
), )
else:
host_option = None
channel = grpc.insecure_channel(target='{host}:{port}'.format(host=host,
port=port),
options=host_option)
stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
with open(input_path) as json_file:
data = json.load(json_file)
image = data['instances'][0]['image_bytes']['b64']
key = data['instances'][0]['key']
# Call classification model to make prediction
request = predict_pb2.PredictRequest()
request.model_spec.name = model
request.model_spec.signature_name = signature_name
image = base64.b64decode(image)
request.inputs['image_bytes'].CopyFrom(make_tensor_proto(image, shape=[1]))
request.inputs['key'].CopyFrom(make_tensor_proto(key, shape=[1]))
result = stub.Predict(request, 10.0)
print(result)
def run(host, port, model, signature_name):
channel = grpc.insecure_channel('{host}:{port}'.format(host=host,
port=port))
stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
start = time.time()
# Call classification model to make prediction
request = predict_pb2.PredictRequest()
request.model_spec.name = model
request.model_spec.signature_name = signature_name
request.inputs['sepal_length'].CopyFrom(
make_tensor_proto(6.8, shape=[1, 1]))
request.inputs['sepal_width'].CopyFrom(make_tensor_proto(3.2, shape=[1,
1]))
request.inputs['petal_length'].CopyFrom(
make_tensor_proto(5.9, shape=[1, 1]))
request.inputs['petal_width'].CopyFrom(make_tensor_proto(2.3, shape=[1,
1]))
result = stub.Predict(request, 10.0)
end = time.time()
time_diff = end - start
# Reference:
# How to access nested values
# https://stackoverflow.com/questions/44785847/how-to-retrieve-float-val-from-a-predictresponse-object
print(result)
print('time elapased: {}'.format(time_diff))
def get_outputs(self, input_data):
# Step 0: check the type of input parameters
if not isinstance(self.ns.host, str):
print("The type of \"host\" must be str (string)!")
raise IllegalArgumentException
if not re_match("^[0-9localhost.:/]+$", self.ns.host):
print("hostport does not match preseted character-set" )
raise IllegalArgumentException
if not isinstance(self.ns.port, int):
print("The type of \"port\* must be int!")
raise IllegalArgumentException
if not isinstance(self.ns.model_name, str):
print("the type of \"model_name\" must be str (string)!")
raise IllegalArgumentException
if not re_match("^[0-9A-Za-z_. \-/]+$", self.ns.model_name):
print("model_name does not match preseted character-set" )
raise IllegalArgumentException
if not isinstance(input_data, dict):
print("the type of \"input_data\" must be dict!")
raise IllegalArgumentException
if (not isinstance(MAX_RESPONSE_TIME, int)) and (not isinstance(MAX_RESPONSE_TIME, float)):
print("the type of \"max_response_time\" must be int or float!")
raise IllegalArgumentException
# Setup connection
channel = implementations.insecure_channel(self.ns.host, self.ns.port)
stub = prediction_service_pb2.beta_create_PredictionService_stub(channel)
# Initialize the request
request = predict_pb2.PredictRequest()
request.model_spec.name = self.ns.model_name
request.model_spec.signature_name = self.ns.model_signature_name
#request.model_spec.version = self.ns.model_version_num
# Set the input variables of the request
for key, value in input_data.items():
if not re_match("^[0-9A-Za-z_. \-/]+$", key):
print("model_name does not match preseted character-set" )
raise IllegalArgumentException
if isinstance(value, numpy_ndarray):
request.inputs[key].CopyFrom(make_tensor_proto(value, shape=list(value.shape)))
elif isinstance(value, int) or isinstance(value, float):
request.inputs[key].CopyFrom(make_tensor_proto(value) )
else:
request.inputs[key].CopyFrom(make_tensor_proto(value, shape=list(value.shape)))
# Obtain the result of prediction
response = stub.Predict(request, MAX_RESPONSE_TIME)
if PRINT_RESPONSE:
responseDict = self.print_response(response)
return responseDict
def process(self, image):
request = predict_pb2.PredictRequest()
request.model_spec.name = self.model_spec_name
request.inputs['in'].CopyFrom(make_tensor_proto(image, dtype='float32'))
request.inputs['min_size'].CopyFrom(make_tensor_proto(self.min_size, dtype='float32'))
request.inputs['factor'].CopyFrom(make_tensor_proto(self.factor, dtype='float32'))
request.inputs['thresholds'].CopyFrom(make_tensor_proto(self.thresholds))
result = self.stub.Predict(request, 10.0)
bbox_np = self.response_to_np(result, 'box')
score_np = self.response_to_np(result, 'prob')
faces_list = self.extract_faces(image, bbox_np.clip(min=0), score_np, config.IMAGE_SIZE)
return faces_list
def send_inference_request(self, image_list):
"""
input: image should be numpy array
"""
request = predict_pb2.PredictRequest()
request.model_spec.name = self.model_spec_name
request.inputs['in'].CopyFrom(
make_tensor_proto(image_list, dtype='float32'))
request.inputs['phase'].CopyFrom(make_tensor_proto(False,
dtype='bool'))
result = self.stub.Predict(request, 10.0) # 10.0s timeout
embeddings_np = self.response_to_np(result, 'out')
return embeddings_np
def main():
# parse command line arguments
host, port, image, image_path, batch_mode = parse_args()
channel = implementations.insecure_channel(host, int(port))
stub = prediction_service_pb2.beta_create_PredictionService_stub(channel)
imagedata = []
if len(image_path) > 0:
filenames = [(image_path + '/' + f) for f in listdir(image_path)
if isfile(join(image_path, f))]
for filename in filenames:
with open(filename, 'rb') as f:
data = f.read()
imagedata.append(data)
else:
with open(image, 'rb') as f:
data = f.read()
imagedata.append(data)
start = time.time()
if batch_mode:
print('In batch mode')
request = predict_pb2.PredictRequest()
request.model_spec.name = 'documents'
request.model_spec.signature_name = 'serving_default'
request.inputs['key'].CopyFrom(
tf.contrib.util.make_tensor_proto("0", shape=[1]))
request.inputs['image_bytes'].CopyFrom(
make_tensor_proto(imagedata, shape=[len(imagedata)]))
result = stub.Predict(request, 60.0)
print(result)
else:
print('In one-by-one mode')
for data in imagedata:
request = predict_pb2.PredictRequest()
request.model_spec.name = 'documents'
request.model_spec.signature_name = 'serving_default'
request.inputs['key'].CopyFrom(
tf.contrib.util.make_tensor_proto("0", shape=[1]))
request.inputs['image_bytes'].CopyFrom(
make_tensor_proto(data, shape=[1]))
result = stub.Predict(request, 60.0) # 60 secs timeout
print(result)
end = time.time()
time_diff = end - start
print('time elapased: {}'.format(time_diff))
def main():
# parse command line arguments
host, port, image_path, batch_mode = parse_args()
channel = implementations.insecure_channel(host, int(port))
stub = prediction_service_pb2.beta_create_PredictionService_stub(channel)
filenames = [(image_path + '/' + f) for f in listdir(image_path)
if isfile(join(image_path, f))]
files = []
imagedata = []
for filename in filenames:
f = open(filename, 'rb')
files.append(f)
data = f.read()
imagedata.append(data)
start = time.time()
if batch_mode:
print('In batch mode')
request = predict_pb2.PredictRequest()
#name of the model trained and exported for our specific purpose
#model name will differ based on the model you used (example: use model from tensorflow model zoo and give name as mentioned there)
request.model_spec.name = 'mnist'
request.model_spec.signature_name = 'predict_images' #signature name as per exporter.py file
request.inputs['images'].CopyFrom(
make_tensor_proto(imagedata, shape=[len(imagedata)]))
result = stub.Predict(request, 60.0) # 60 secs timeout
print(result)
else:
print('In one-by-one mode')
for data in imagedata:
request = predict_pb2.PredictRequest()
request.model_spec.name = 'mnist'
request.model_spec.signature_name = 'predict_images'
request.inputs['images'].CopyFrom(
make_tensor_proto(data, shape=[1]))
result = stub.Predict(request, 60.0) # 60 secs timeout
print(result)
end = time.time()
time_diff = end - start
print('time elapased: {}'.format(time_diff))
def post(self):
json_data = request.get_json(force=True)
image_path = json_data['image_path']
model_name = json_data['model_name']
for inx, i in enumerate(dictionary_data):
i['model_name'] == model_name
break
model_data = dictionary_data[inx]
signature_name = model_data['signature_name']
inputs = model_data['inputs']
outputs = model_data['outputs']
outputs_type = model_data['outputs_type']
image_data = image_preprocessing(model_name, image_path)
# 예측 !!
request_tensor.model_spec.name = model_name
request_tensor.model_spec.signature_name = signature_name
request_tensor.inputs[inputs].CopyFrom(
make_tensor_proto(image_data, shape=list(image_data.shape)))
result_predict = stub.Predict(request_tensor, 1000.0)
print("-----result-----")
print(result_predict)
final = {}
final['결과'] = list(
type_val(result_predict,
output_name=outputs,
output_type=outputs_type))
return final
def get_model_output(data):
'''
Call tensorflow model server through grpc for model output
'''
host, port, model, signature_name = \
model_params.tensorflow_host, model_params.tensorflow_port, \
model_params.tensorflow_model, model_params.tensorflow_signature_name
channel = grpc.insecure_channel('{host}:{port}'.format(host=host, port=port))
stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
start = time.time()
# Call classification model to make prediction on the image
request = predict_pb2.PredictRequest()
request.model_spec.name = model
request.model_spec.signature_name = signature_name
request.inputs['input'].CopyFrom(make_tensor_proto(data['grid_table'], shape=data['grid_table'].shape))
result = stub.Predict(request, 10.0)
outputs_tensor_proto = result.outputs['output']
shape = tf.TensorShape(outputs_tensor_proto.tensor_shape)
outputs = np.array(outputs_tensor_proto.float_val).reshape(shape.as_list())
return outputs
def run(inputs):
beg = time.time()
# 获取输入
pinyin = inputs['PinYin']
# 处理数据
# pinyin = ['jin1','tian1','tian1','qi4','zhen1','hao3']
pinyin = [lm_pinyin_vocab.index(i) for i in pinyin]
pinyin = np.asarray(pinyin).reshape(1, -1)
# print(pinyin.shape, pinyin)
host, port = server.split(':')
channel = implementations.insecure_channel(host, int(port))
# stub = prediction_service_pb2.beta_create_PredictionService_stub(channel)
stub = prediction_service_pb2_grpc.PredictionServiceStub(
channel._channel)
request = predict_pb2.PredictRequest()
request.model_spec.name = model_name
request.model_spec.signature_name = 'predict_Pinyin2Hanzi'
request.inputs['pinyin'].CopyFrom(
make_tensor_proto(pinyin, shape=pinyin.shape,
dtype='int32')) # 1是batch_size
result = stub.Predict(request, 60.0)
pred_label = np.array(result.outputs['hanzi'].int_val)
# print(pred_label.shape, pred_label)
hanzi = [lm_hanzi_vocab[i] for i in pred_label]
# print( bytes("".join(hanzi), encoding = "utf8"))
# 返回
ret = {'HanZi': "".join(hanzi)}
print('LM_serving Time:', time.time() - beg)
return ret
def main():
# parse command line arguments
host, port, test_data = parse_args()
channel = implementations.insecure_channel(host, int(port))
stub = prediction_service_pb2.beta_create_PredictionService_stub(channel)
# Send request
# See prediction_service.proto for gRPC request/response details.
start = time.time()
request = predict_pb2.PredictRequest()
# Call GAN model to make prediction on the image
request.model_spec.name = 'base'
request.model_spec.signature_name = 'predict'
print(test_data.shape)
request.inputs['states'].CopyFrom(
make_tensor_proto(test_data, dtype=tf.float32))
result = stub.Predict(request, 60.0) # 60 secs timeout
end = time.time()
time_diff = end - start
print(result)
print('time elapased: {}'.format(time_diff))
def main():
# parse command line arguments
host, port, image = parse_args()
channel = implementations.insecure_channel(host, int(port))
stub = prediction_service_pb2.beta_create_PredictionService_stub(channel)
# Send request
with open(image, 'rb') as f:
# See prediction_service.proto for gRPC request/response details.
data = f.read()
start = time.time()
request = predict_pb2.PredictRequest()
# Call GAN model to make prediction on the image
request.model_spec.name = 'gan'
request.model_spec.signature_name = 'predict_images'
request.inputs['images'].CopyFrom(make_tensor_proto(data, shape=[1]))
result = stub.Predict(request, 60.0) # 60 secs timeout
end = time.time()
time_diff = end - start
print(result)
print('time elapased: {}'.format(time_diff))
def main():
# Parse command line arguments
host, port = parse_args()
# Create a grpc channel using the IP adress and the port
channel = implementations.insecure_channel(host, int(port))
# Create a stub
stub = prediction_service_pb2.beta_create_PredictionService_stub(channel)
# Name of the data that should be send to the server
file = '../data/raw/creditcard_dataset_fraud.csv'
data = get_data(data_type="test", data_dir=file)
sample_df = data.sample(20)
for _, row in sample_df.iterrows():
input_data = [[s for s in row]]
# print(input_data)
# Create a request object
request = predict_pb2.PredictRequest()
# Name of the model running on the tensorflow_model_server (either locally or in Docker container)
request.model_spec.name = 'anamoly_detection'
# Name of the defined prediction signature in the SavedModelInstance on the server (either locally or in Docker container)
request.model_spec.signature_name = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
# Make a request (time-out after 20 seconds)
request.inputs['inputs'].CopyFrom(
make_tensor_proto(input_data, shape=[1, 29]))
result = stub.Predict(request, 20.0) # 60 secs timeout
print(result)
def create_warmup_requests_numpy(self, batch_sizes, export_dir):
"""Creates warm-up requests for a given feature specification.
This writes an output file in
`export_dir/assets.extra/tf_serving_warmup_requests` for use with Servo.
Args:
batch_sizes: Batch sizes of warm-up requests to write.
export_dir: Base directory for the export.
Returns:
The filename written.
"""
feature_spec = self._get_input_features_for_receiver_fn()
flat_feature_spec = tensorspec_utils.flatten_spec_structure(
feature_spec)
tf.io.gfile.makedirs(export_dir)
request_filename = os.path.join(export_dir,
'tf_serving_warmup_requests')
with tf.python_io.TFRecordWriter(request_filename) as writer:
for batch_size in batch_sizes:
request = predict_pb2.PredictRequest()
request.model_spec.name = self._model_name
numpy_feature_specs = tensorspec_utils.make_constant_numpy(
flat_feature_spec, constant_value=0, batch_size=batch_size)
for key, numpy_spec in numpy_feature_specs.items():
request.inputs[key].CopyFrom(
contrib_util.make_tensor_proto(numpy_spec))
log = prediction_log_pb2.PredictionLog(
predict_log=prediction_log_pb2.PredictLog(request=request))
writer.write(log.SerializeToString())
return request_filename
def predict_test1(batch_size, serving_config, img):
channel = grpc.insecure_channel(serving_config['hostport'],
options=[
('grpc.max_send_message_length',
serving_config['max_message_length']),
('grpc.max_receive_message_length',
serving_config['max_message_length'])
])
stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
#img = cv2.imdecode(image, cv2.IMREAD_COLOR)
#cv2.imshow("Image", image)
# Test on some demo image and visualize output.
#image_path = os.path.join(os.getcwd(),"images/")
#image_names = sorted(os.listdir(image_path))
#print(image_names)
#img = mpimg.imread(image_path + image_names[1])
# Creating random images for given batch size
#input_data=np.ones((500,500,3),dtype="uint8")
input_data = img
print(img.shape)
request = predict_pb2.PredictRequest()
request.model_spec.name = serving_config['model_name']
request.model_spec.signature_name = serving_config['signature_name']
request.inputs['input0'].CopyFrom(
make_tensor_proto(input_data,
shape=img.shape,
dtype=types_pb2.DT_UINT8))
result = stub.Predict(request, serving_config['timeout'])
channel.close()
return result
def make_request(image_path, server):
"""
:param image_path:
:param server:
:return:
"""
channel = grpc.insecure_channel(server)
stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
image = cv2.imread(image_path, cv2.IMREAD_COLOR)
image = cv2.resize(image, (CFG.ARCH.INPUT_SIZE[0], CFG.ARCH.INPUT_SIZE[1]),
interpolation=cv2.INTER_LINEAR)
image = np.array(image, np.float32) / 127.5 - 1.0
image_list = np.array([image], dtype=np.float32)
request = predict_pb2.PredictRequest()
request.model_spec.name = 'crnn'
request.model_spec.signature_name = 'crnn_prediction_result'
request.inputs['input_tensor'].CopyFrom(
make_tensor_proto(
image_list,
shape=[1, CFG.ARCH.INPUT_SIZE[1], CFG.ARCH.INPUT_SIZE[0], 3]))
try:
result = stub.Predict(request, 10.0)
return result
except Exception as err:
print(err)
return None
def run(self, image, width, height):
"""image: bgr image
return (boxes, scores, classes, num_detections)
"""
request = predict_pb2.PredictRequest()
request.model_spec.name = self.model_name
request.model_spec.signature_name = self.signature_name
request.model_spec.version.value = int(self.version)
request.inputs['image'].CopyFrom(
make_tensor_proto(image, shape=[1, width, height, 3]))
result = self.stub.Predict(request)
boxes, scores, classes, num_detections = result.outputs[
'boxes'].float_val, result.outputs[
'scores'].float_val, result.outputs[
'classes'].float_val, result.outputs[
'num_detections'].float_val
num_detections = int(num_detections[0])
boxes = np.reshape(boxes, [1, num_detections, 4])
scores = np.reshape(scores, [1, num_detections])
classes = np.reshape(classes, [1, num_detections])
return boxes, scores, classes, num_detections
def run(host, port, image, model, signature_name):
# channel = grpc.insecure_channel('%s:%d' % (host, port))
channel = implementations.insecure_channel(host, port)
stub = prediction_service_pb2.beta_create_PredictionService_stub(channel)
# Read an image
data = imread(image)
data = data.astype(np.float32)
start = time.time()
# Call classification model to make prediction on the image
request = predict_pb2.PredictRequest()
request.model_spec.name = model
request.model_spec.signature_name = signature_name
request.inputs['image'].CopyFrom(
make_tensor_proto(data, shape=[1, 28, 28, 1]))
result = stub.Predict(request, 10.0)
end = time.time()
time_diff = end - start
# Reference:
# How to access nested values
# https://stackoverflow.com/questions/44785847/how-to-retrieve-float-val-from-a-predictresponse-object
print(result)
print('time elapased: {}'.format(time_diff))
def main(_):
channel = grpc.insecure_channel(FLAGS.server)
stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
request = predict_pb2.PredictRequest()
request.model_spec.name = FLAGS.model
request.model_spec.signature_name = "serving_default"
img = cv2.imread(FLAGS.img)
retr, buffer = cv2.imencode(".jpg", img)
jpg_str = base64.urlsafe_b64encode(buffer).decode("utf-8")
difference_sum = 0
for i in range(FLAGS.num_tests):
request.inputs['image'].CopyFrom(make_tensor_proto(jpg_str,shape=[1,]))
start = time.time()
response = stub.Predict(request,10)
diff = time.time() - start
print("Predict num: {} consumed time: {}".format(i,diff))
difference_sum += diff
if FLAGS.vis:
classes = np.squeeze(make_ndarray(response.outputs["class"]))
box = np.squeeze(make_ndarray(response.outputs["box"]))
confidence = np.squeeze(make_ndarray(response.outputs["confidence"]))
visimg = cv2.imread(FLAGS.img)
for o in range(len(classes)):
draw_boxes(visimg,box[o],confidence[o],classes[o])
cv2.imshow("Vis",visimg)
cv2.waitKey(1)
print("Average time: {}".format(difference_sum / FLAGS.num_tests))
def do_inference(hostport, work_dir, concurrency, num_tests):
"""Tests PredictionService with concurrent requests.
Args:
hostport: Host:port address of the PredictionService.
work_dir: The full path of working directory for test data set.
concurrency: Maximum number of concurrent requests.
num_tests: Number of test images to use.
Returns:
The classification error rate.
Raises:
IOError: An error occurred processing test data set.
"""
test_data_set = mnist_input_data.read_data_sets(work_dir).test
channel = grpc.insecure_channel(hostport)
stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
result_counter = _ResultCounter(num_tests, concurrency)
for _ in range(num_tests):
request = predict_pb2.PredictRequest()
request.model_spec.name = 'mnist'
request.model_spec.signature_name = 'predict_images'
image, label = test_data_set.next_batch(1)
request.inputs['images'].CopyFrom(
contrib_util.make_tensor_proto(image[0], shape=[1, image[0].size]))
result_counter.throttle()
result_future = stub.Predict.future(request, 5.0) # 5 seconds
result_future.add_done_callback(
_create_rpc_callback(label[0], result_counter))
return result_counter.get_error_rate()
def main(args):
host, port = args.host, args.port
mnist = input_data.read_data_sets(args.data_dir, one_hot=True)
channel = implementations.insecure_channel(host, int(port))
stub = prediction_service_pb2.beta_create_PredictionService_stub(channel)
start_time = time.time()
counter = 0
num_tests = args.num_tests
time
for _ in range(num_tests):
image, label = mnist.test.next_batch(1)
request = predict_pb2.PredictRequest()
request.model_spec.name = 'mnist'
request.model_spec.signature_name = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
request.inputs['x_input'].CopyFrom(
make_tensor_proto(image[0], shape=[1, 28, 28]))
result_future = stub.Predict(request, 10.0)
exception = result_future.exception()
if exception:
print(exception)
else:
response = np.array(result_future.outputs['y_output'].float_val)
if np.argmax(response) == np.argmax(label):
counter += 1
else:
pass
print("Accuracy= %0.2f" % ((counter * 1.0 / num_tests) * 100))
print("Time takes to run the test %0.2f" % (time.time() - start_time))
def run(host, port, image_filepath, model, signature_name):
channel = grpc.insecure_channel('{host}:{port}'.format(host=host, port=port))
stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
# Read an image
data = np.load(image_filepath)
data = data.reshape((1, 28, 28, 1))
data = data.astype(np.float32) / 255.
start = time.time()
# Call classification model to make prediction on the image
request = predict_pb2.PredictRequest()
request.model_spec.name = model
request.model_spec.signature_name = signature_name
request.inputs['input'].CopyFrom(make_tensor_proto(data, dtype=np.float32, shape=[1, 28, 28, 1], verify_shape=True))
result = stub.Predict(request, 10.0)
end = time.time()
time_diff = end - start
# Reference:
# How to access nested values
# https://stackoverflow.com/questions/44785847/how-to-retrieve-float-val-from-a-predictresponse-object
print('time elapased: {}'.format(time_diff))
print(result)
print('')
print('Predicted class: {}'.format(result.outputs['classes'].int64_val))
print('Probabilities: {}'.format(result.outputs['probabilities'].float_val))
def generate_tokens(input_tokens):
input_data = np.array([input_tokens], dtype='int32')
request.inputs['input'].CopyFrom(make_tensor_proto(input_data,
shape=input_data.shape))
response = stub.Predict(request, timeout)
tensor = response.outputs['output']
return [tensor.int_val]
def get_fake_request(model_name, data_shape, input_blob, version=None):
request = predict_pb2.PredictRequest()
request.model_spec.name = model_name
if version is not None:
request.model_spec.version.value = version
data = np.ones(shape=data_shape)
request.inputs[input_blob].CopyFrom(
make_tensor_proto(data, shape=data.shape))
return request
def get_predictions(host, port, model_name, signature_name, input_placeholder_name, mat):
channel = implementations.insecure_channel(host, port)._channel
stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
request = predict_pb2.PredictRequest()
request.model_spec.name = model_name
request.model_spec.signature_name = signature_name
request.inputs[input_placeholder_name].CopyFrom(make_tensor_proto(mat))
result = stub.Predict(request)
return result
def __create_service_request(self, mr_encoded):
"""Assembles a request for the prediction service."""
request = predict_pb2.PredictRequest()
request.model_spec.name = self.model_name
request.inputs['input'].CopyFrom(
make_tensor_proto([mr_encoded], shape=[1]))
return request
def main():
tokenizer = tokenization.FullTokenizer(
vocab_file='./albert_config/vocab.txt', do_lower_case=True)
begin = time.time()
sentences = []
sentences_1 = ("您好.麻烦您截图全屏辛苦您了.", "麻烦您截图大一点辛苦您了.最好可以全屏.")
for i in range(600):
sentences.append(sentences_1)
input_ids = []
input_mask = []
label_ids = []
segment_ids = []
for i in sentences:
features = prcoess_data(i, tokenizer)
input_ids.append(features.input_ids)
input_mask.append(features.input_mask)
label_ids.append([0])
segment_ids.append(features.segment_ids)
server = '0.0.0.0:9000'
channel = grpc.insecure_channel(server)
stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
request = predict_pb2.PredictRequest()
request.model_spec.name = 'test'
request.model_spec.signature_name = 'serving_default'
input_ids = util.make_tensor_proto(input_ids, shape=[len(sentences), 64])
input_mask = util.make_tensor_proto(input_mask, shape=[len(sentences), 64])
label_ids = util.make_tensor_proto(label_ids, shape=[len(sentences), 1])
segment_ids = util.make_tensor_proto(segment_ids,
shape=[len(sentences), 64])
request.inputs['input_ids'].CopyFrom(input_ids)
request.inputs['input_mask'].CopyFrom(input_mask)
request.inputs['label_ids'].CopyFrom(label_ids)
request.inputs['segment_ids'].CopyFrom(segment_ids)
result = stub.Predict(request, 10.0) # 10 secs timeout
end = time.time() - begin
output = np.array(result.outputs['probabilities'].float_val)
print(len(sentences) / 2)
print('time {}'.format(end))
def inference(stub, request, imgs, kwargs):
print("Start processing:")
print('\tModel name: {}'.format(kwargs['model_name']))
print('\tImages in shape: {}\n'.format(imgs.shape))
processing_times = np.zeros((0), int)
batch_size = int(kwargs['batch_size'])
if not (kwargs.get('images_number') or kwargs.get('images_number') != 0):
iterations = int((imgs.shape[0] // batch_size))
else:
iterations = int(kwargs.get('images_number'))
iteration = 0
while iteration <= iterations:
for x in range(0, imgs.shape[0] - batch_size + 1, batch_size):
iteration += 1
if iteration > iterations:
break
end_batch = x + batch_size
if end_batch > imgs.shape[0]:
end_batch = imgs.shape[0]
batch = imgs[x:end_batch]
request.inputs[kwargs['input_name']].CopyFrom(
tf_contrib_util.make_tensor_proto(batch, shape=(batch.shape)))
start_time = datetime.datetime.now()
result = stub.Predict(request, RPC_TIMEOUT)
# result includes a dictionary with all model outputs
end_time = datetime.datetime.now()
if kwargs['output_name'] not in result.outputs:
print("Invalid output name", kwargs['output_name'])
print("Available outputs:")
for Y in result.outputs:
print(Y)
exit(1)
duration = (end_time - start_time).total_seconds() * 1000
processing_times = \
np.append(processing_times, np.array([int(duration)]))
output = \
tf_contrib_util.make_ndarray(result.outputs[kwargs['output_name']])
print('Iteration {}; Processing time: {:.2f} ms; speed {:.2f} fps'.
format(iteration, round(np.average(duration), 2),
round(1000 * batch_size / np.average(duration), 2)))
if kwargs['no_imagenet_classes']:
continue
print('\tImagenet top results in a single batch:')
for i in range(output.shape[0]):
single_result = output[[i], ...]
max_class = np.argmax(single_result)
print('\t\t {} image in batch: {}'.format(
i + 1, classes.imagenet_classes[max_class]))
if kwargs['performance']:
get_processing_performance(processing_times, batch_size)
return output
