def build_request(self, input):
request = grpc_service_pb2.InferRequest()
request.model_name = self.model_name
request.model_version = self.model_version
batch_size = self.model.input[0].dims[0]
request.meta_data.batch_size = batch_size
for output in self.model.output:
request.meta_data.output.add(name=output.name)
# output_message = api_pb2.InferRequestHeader.Output()
# output_message.name = output.name
# request.meta_data.output.extend([output_message])
del request.raw_input[:]
for name, value in input.items():
if type(value) in [bytes, str]:
input_bytes, byte_size = self._to_bytes(value)
request.meta_data.input.add(
name=name, dims=[1],
batch_byte_size=byte_size
)
request.raw_input.append(input_bytes)
elif type(value) is np.ndarray:
input_bytes = value.tobytes()
batch_byte_size = len(input_bytes)
request.meta_data.input.add(
name=name, dims=list(value.shape),
batch_byte_size=batch_byte_size
)
request.raw_input.append(input_bytes)
#print(request)
return request
def prepare_request(self, img):
"""
Prepare gRPC request for model inference with the tensorrt inference server.
Arguments:
img(PIL.Image): Input image.
Returns:
request(gRPC Request object): Request object with all information about the request.
"""
request = grpc_service_pb2.InferRequest()
request.model_name = self._MODEL_NAME
request.meta_data.batch_size = self._BATCH_SIZE
request.model_version = self._MODEL_VERSION
output_list = ["base_network/lambda_1/l2_normalize"]
for output in output_list:
output_message = api_pb2.InferRequestHeader.Output()
output_message.name = output
request.meta_data.output.extend([output_message])
image_data = []
image_data.append(self.fashion_matching_preprocess(img))
request.meta_data.input.add(name="query_input", dims=[32, 32, 3])
input_bytes = image_data[0].tobytes()
request.raw_input.extend([input_bytes])
return request
def compose_request(input_name, output_name, c, w, h, model_name, model_version):
'''
Compose gRPC request for different models.
Arguments :
input_name : input placeholder for model
output_name : output placeholders for model ( one or more)
c : channel
w : width
h : height
model_name : model name to connect
model_version: model version
Returns:
gRPC request
'''
request = grpc_service_pb2.InferRequest()
request.model_name = model_name
request.model_version = model_version
request.meta_data.batch_size = 1
request.meta_data.input.add(dims=[w,h,c], name=input_name)
output_names = output_name.split(',')
out_len = len(output_names)
if out_len > 1:
for i in range(out_len):
output_message = api_pb2.InferRequestHeader.Output()
output_message.name = output_names[i]
request.meta_data.output.extend([output_message])
else:
output_message = api_pb2.InferRequestHeader.Output()
output_message.name = output_name
request.meta_data.output.extend([output_message])
return request
def requestGenerator(input_name, output_name, c, h, w, format, dtype, FLAGS, result_filenames):
# Prepare request for Infer gRPC
# The meta data part can be reused across requests
request = grpc_service_pb2.InferRequest()
request.model_name = FLAGS.model_name
if FLAGS.model_version is None:
request.model_version = -1
else:
request.model_version = FLAGS.model_version
request.meta_data.batch_size = FLAGS.batch_size
output_message = api_pb2.InferRequestHeader.Output()
output_message.name = output_name
output_message.cls.count = FLAGS.classes
request.meta_data.output.extend([output_message])
filenames = []
if os.path.isdir(FLAGS.image_filename):
filenames = [os.path.join(FLAGS.image_filename, f)
for f in os.listdir(FLAGS.image_filename)
if os.path.isfile(os.path.join(FLAGS.image_filename, f))]
else:
filenames = [FLAGS.image_filename,]
filenames.sort()
# Preprocess the images into input data according to model
# requirements
image_data = []
for filename in filenames:
img = Image.open(filename)
image_data.append(preprocess(img, format, dtype, c, h, w, FLAGS.scaling))
request.meta_data.input.add(name=input_name)
# Send requests of FLAGS.batch_size images. If the number of
# images isn't an exact multiple of FLAGS.batch_size then just
# start over with the first images until the batch is filled.
image_idx = 0
last_request = False
while not last_request:
input_bytes = None
input_filenames = []
del request.raw_input[:]
for idx in range(FLAGS.batch_size):
input_filenames.append(filenames[image_idx])
if input_bytes is None:
input_bytes = image_data[image_idx].tobytes()
else:
input_bytes += image_data[image_idx].tobytes()
image_idx = (image_idx + 1) % len(image_data)
if image_idx == 0:
last_request = True
request.raw_input.extend([input_bytes])
result_filenames.append(input_filenames)
yield request
def requestGenerator(self, output_name, FLAGS):
# Prepare request for Infer gRPC
# The meta data part can be reused across requests
request = grpc_service_pb2.InferRequest()
request.model_name = self.model_name
request.model_version = -1
request.meta_data.batch_size = 1
output_message = api_pb2.InferRequestHeader.Output()
output_message.name = output_name
request.meta_data.output.extend([output_message])
input0_data = np.arange(start=0, stop=16, dtype=np.int32)
input_bytes = input0_data.tobytes()
request.meta_data.input.add(name="INPUT0", dims=[16])
del request.raw_input[:]
request.raw_input.extend([input_bytes])
return request
def init_request(model_name, data_dir, data_size):
image_id = numpy.random.randint(data_size) + 1
image_file = '{}/{:0>5}.jpg'.format(data_dir, image_id)
image = Image.open(image_file).convert('RGB')
image = image.resize((224, 224), Image.BILINEAR)
image = numpy.array(image).astype(numpy.float32)
input_bytes = image.tobytes()
request = grpc_service_pb2.InferRequest()
request.model_name = model_name
request.model_version = -1
request.meta_data.batch_size = 1
output_message = api_pb2.InferRequestHeader.Output()
output_message.name = 'probabilities'
output_message.cls.count = 1
request.meta_data.output.extend([output_message])
request.meta_data.input.add(name='input')
request.raw_input.extend([input_bytes])
return request
def prepare_request(self, img):
"""
Prepare gRPC request for model inference with the tensorrt inference server.
Arguments:
img(PIL.Image): Input image.
Returns:
request(gRPC Request object): Request object with all information about the request.
"""
request = grpc_service_pb2.InferRequest()
request.model_name = self._MODEL_NAME
request.model_version = self._MODEL_VERSION
request.meta_data.batch_size = self._BATCH_SIZE
output_message = api_pb2.InferRequestHeader.Output()
output_message.name = "vgg0_dense2_fwd"
request.meta_data.output.extend([output_message])
image_data = []
image_data.append(self.vgg16_preprocess(img))
request.meta_data.input.add(name="data")
input_bytes = image_data[0].tobytes()
request.raw_input.extend([input_bytes])
return request
# Create gRPC stub for communicating with the server
channel = grpc.insecure_channel(FLAGS.url)
grpc_stub = grpc_service_pb2_grpc.GRPCServiceStub(channel)
# Prepare request for Status gRPC
request = grpc_service_pb2.StatusRequest(model_name=FLAGS.model_name)
# Call and receive response from Status gRPC
response = grpc_stub.Status(request)
# Make sure the model matches our requirements, and get some
# properties of the model that we need for preprocessing
input_name, output_name, c, h, w, format, dtype, output_size = parse_model(
response, FLAGS.model_name, FLAGS.batch_size, FLAGS.verbose)
# Prepare request for Infer gRPC
# The meta data part can be reused across requests
request = grpc_service_pb2.InferRequest()
request.model_name = FLAGS.model_name
if FLAGS.model_version is None:
request.model_version = -1
else:
request.model_version = FLAGS.model_version
request.meta_data.batch_size = FLAGS.batch_size
output_message = api_pb2.InferRequestHeader.Output()
output_message.name = output_name
output_message.byte_size = output_size
output_message.cls.count = FLAGS.classes
request.meta_data.output.extend([output_message])
filenames = []
if os.path.isdir(FLAGS.image_filename):
filenames = [
