def _grpc_client(self): if self._grpc_client_stub is None: # requires appending ":80" to the predictor host for gRPC to work if ":" not in self.predictor_host: self.predictor_host = self.predictor_host + ":80" _channel = grpc.aio.insecure_channel(self.predictor_host) self._grpc_client_stub = service_pb2_grpc.GRPCInferenceServiceStub( _channel) return self._grpc_client_stub
default='localhost:8001', help='Inference server URL. Default is localhost:8001.') FLAGS = parser.parse_args() # We use a simple model that takes 2 input tensors of 16 integers # each and returns 2 output tensors of 16 integers each. One # output tensor is the element-wise sum of the inputs and one # output is the element-wise difference. model_name = "simple_int8" model_version = "" batch_size = 1 # Create gRPC stub for communicating with the server channel = grpc.insecure_channel(FLAGS.url) grpc_stub = service_pb2_grpc.GRPCInferenceServiceStub(channel) # Generate the request request = service_pb2.ModelInferRequest() request.model_name = model_name request.model_version = model_version # Input data input0_data = [i for i in range(16)] input1_data = [1 for i in range(16)] # Populate the inputs in inference request input0 = service_pb2.ModelInferRequest().InferInputTensor() input0.name = "INPUT0" input0.datatype = "INT8" input0.shape.extend([1, 16])