def response_thread(self, response_sender, index, in_input):
# The response_sender is used to send response(s) associated with the
# corresponding request. The first request will send errors and the
# other requests will send the responses. The number of responses per
# requests is the number of elements in input tensor.
in_value = in_input
out_output = pb_utils.Tensor("OUT", in_value)
if index == 0:
error = pb_utils.TritonError('An error occured during execution')
response = pb_utils.InferenceResponse(output_tensors=[out_output],
error=error)
else:
response = pb_utils.InferenceResponse(output_tensors=[out_output])
response_sender.send(response)
# We must close the response sender to indicate to Triton that we are
# done sending responses for the corresponding request. We can't use the
# response sender after closing it.
response_sender.send(
flags=pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL)
with self.inflight_thread_count_lck:
self.inflight_thread_count -= 1
def execute(self, requests):
responses = []
for request in requests:
in0 = pb_utils.get_input_tensor_by_name(request, "PYTHON_INPUT_0")
in0_t = in0.as_numpy()
decoded = []
for inp in in0_t:
aud_sr = decode_audio(inp.tobytes())
decoded.append((aud_sr[0], aud_sr[0].shape[0]))
max_len = 0
for dec in decoded:
max_len = max_len if max_len > dec[1] else dec[1]
audio = []
audio_lens = []
for aud, length in decoded:
audio.append(aud)
np.pad(audio[-1], (0, max_len - audio[-1].shape[0]))
audio_lens.append(length)
audio_array = np.array(audio)
len_array = np.array(audio_lens)
dec_t = torch.Tensor(audio_array)
len_t = torch.Tensor(len_array)
dec_t = dec_t.cuda()
len_t = len_t.cuda()
out_audio, out_len = self.feat_proc(dec_t, len_t)
out0_tensor = pb_utils.Tensor.from_dlpack(
"PYTHON_OUTPUT_0", torch.utils.dlpack.to_dlpack(out_audio))
response = pb_utils.InferenceResponse(output_tensors=[out0_tensor])
responses.append(response)
return responses
def test_infer_response_args(self):
outputs = [
pb_utils.Tensor('OUTPUT0', np.asarray([1, 2], dtype=np.int32))
]
# Test list of None object as output tensor
with self.assertRaises(pb_utils.TritonModelException) as e:
pb_utils.InferenceResponse(output_tensors=[None])
# Test None as output tensors
with self.assertRaises(TypeError) as e:
pb_utils.InferenceResponse(output_tensors=None)
# This should not raise an exception
pb_utils.InferenceResponse(output_tensors=[])
pb_utils.InferenceResponse(outputs)
def execute(self, requests):
"""Model supporting optional inputs. If the input is not provided, an
input tensor of size 1 containing scalar 5 will be used."""
responses = []
for request in requests:
input0_tensor = pb_utils.get_input_tensor_by_name(request, "INPUT0")
input1_tensor = pb_utils.get_input_tensor_by_name(request, "INPUT1")
if input0_tensor is not None:
input0_numpy = input0_tensor.as_numpy()
else:
input0_numpy = np.array([5], dtype=np.int32)
if input1_tensor is not None:
input1_numpy = input1_tensor.as_numpy()
else:
input1_numpy = np.array([5], dtype=np.int32)
output0_tensor = pb_utils.Tensor("OUTPUT0",
input0_numpy + input1_numpy)
output1_tensor = pb_utils.Tensor("OUTPUT1",
input0_numpy - input1_numpy)
responses.append(
pb_utils.InferenceResponse([output0_tensor, output1_tensor]))
return responses
def execute(self, requests):
output0_dtype = self.output0_dtype
responses = []
for request in requests:
in_0 = pb_utils.get_input_tensor_by_name(request, "INPUT0")
input_smiles = in_0.as_numpy()[0].decode()
print('processing', input_smiles)
generated_smiles, neighboring_embeddings, pad_mask = \
self.find_similars_smiles_list(input_smiles,
num_requested=10,
force_unique=True)
out_0 = np.array(generated_smiles).astype(np.object)
out_tensor_0 = pb_utils.Tensor("OUTPUT0",
out_0.astype(output0_dtype))
# pb_utils.InferenceResponse(
# output_tensors=..., TritonError("An error occured"))
inference_response = pb_utils.InferenceResponse(
output_tensors=[out_tensor_0])
responses.append(inference_response)
return responses
def execute(self, requests):
""" Create a response sender object and use that
for sending the response.
"""
# This model does not support batching, so 'request_count' should always be 1.
if len(requests) != 1:
raise pb_utils.TritonModelException("unsupported batch size " +
len(requests))
output0_dtype = self.output0_dtype
output1_dtype = self.output1_dtype
response_sender = requests[0].get_response_sender()
in_0 = pb_utils.get_input_tensor_by_name(requests[0], "INPUT0")
in_1 = pb_utils.get_input_tensor_by_name(requests[0], "INPUT1")
out_0, out_1 = (in_0.as_numpy() + in_1.as_numpy(),
in_0.as_numpy() - in_1.as_numpy())
out_tensor_0 = pb_utils.Tensor("OUTPUT0", out_0.astype(output0_dtype))
out_tensor_1 = pb_utils.Tensor("OUTPUT1", out_1.astype(output1_dtype))
response = pb_utils.InferenceResponse([out_tensor_0, out_tensor_1])
response_sender.send(
flags=pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL)
response_sender.send(response)
def execute(self, requests):
responses = []
for request in requests:
input0 = pb_utils.get_input_tensor_by_name(request, "INPUT0")
print('ISCPU', input0.is_cpu())
gpu_output = pb_utils.get_input_tensor_by_name(
request, "GPU_OUTPUT").as_numpy()
if input0.is_cpu():
if not gpu_output[0]:
output0 = pb_utils.Tensor.from_dlpack(
"OUTPUT0", input0.to_dlpack())
else:
outptu0_pytorch = from_dlpack(input0.to_dlpack()).cuda()
output0 = pb_utils.Tensor.from_dlpack(
"OUTPUT0", to_dlpack(outptu0_pytorch))
else:
if gpu_output[0]:
output0 = pb_utils.Tensor.from_dlpack(
"OUTPUT0", input0.to_dlpack())
else:
outptu0_pytorch = from_dlpack(input0.to_dlpack()).cpu()
output0 = pb_utils.Tensor.from_dlpack(
"OUTPUT0", to_dlpack(outptu0_pytorch))
next_gpu_output = pb_utils.Tensor("NEXT_GPU_OUTPUT",
gpu_output[1:])
responses.append(
pb_utils.InferenceResponse([output0, next_gpu_output]))
return responses
def execute(self, requests):
output0_dtype = self.output0_dtype
output1_dtype = self.output1_dtype
responses = []
for request in requests:
THRESHOLD = 0.20
# Get input
x_recon = pb_utils.get_input_tensor_by_name(
request, "RECONSTR0").as_numpy()
x_orig = pb_utils.get_input_tensor_by_name(request,
"ORIG0").as_numpy()
# Get Mean square error between reconstructed input and original input
reconstruction_score = np.mean((x_orig - x_recon)**2, axis=1)
anomaly = reconstruction_score > THRESHOLD
# Create output tensors
out_tensor_0 = pb_utils.Tensor(
"ANOMALY_SCORE0", reconstruction_score.astype(output0_dtype))
out_tensor_1 = pb_utils.Tensor("ANOMALY0",
anomaly.astype(output1_dtype))
inference_response = pb_utils.InferenceResponse(
output_tensors=[out_tensor_0, out_tensor_1])
responses.append(inference_response)
return responses
def execute(self, requests):
output0_dtype = self.output0_dtype
output1_dtype = self.output1_dtype
responses = []
for request in requests:
in_0 = pb_utils.get_input_tensor_by_name(request, "INPUT0")
in_1 = pb_utils.get_input_tensor_by_name(request, "INPUT1")
# If both of the tensors are in CPU, use NumPy.
if in_0.is_cpu() and in_1.is_cpu():
if in_0.as_numpy().dtype.type is np.bytes_ or in_0.as_numpy(
).dtype == np.object_:
out_0, out_1 = (in_0.as_numpy().astype(np.int32) - in_1.as_numpy().astype(np.int32),\
in_0.as_numpy().astype(np.int32) + in_1.as_numpy().astype(np.int32))
out_tensor_0 = pb_utils.Tensor("OUTPUT0",
out_0.astype(output0_dtype))
out_tensor_1 = pb_utils.Tensor("OUTPUT1",
out_1.astype(output1_dtype))
else:
in_0_pytorch, in_1_pytorch = from_dlpack(
in_0.to_dlpack()), from_dlpack(in_1.to_dlpack())
out_0, out_1 = (in_0_pytorch - in_1_pytorch,
in_0_pytorch + in_1_pytorch)
if self.output0_dtype == np.object_:
out_tensor_0 = pb_utils.Tensor(
"OUTPUT0",
out_0.numpy().astype(output0_dtype))
else:
out_0 = out_0.type(
self.numpy_to_pytorch_dtype[output0_dtype])
out_tensor_0 = pb_utils.Tensor.from_dlpack(
"OUTPUT0", to_dlpack(out_0))
if self.output1_dtype == np.object_:
out_tensor_1 = pb_utils.Tensor(
"OUTPUT1",
out_1.numpy().astype(output1_dtype))
else:
out_1 = out_1.type(
self.numpy_to_pytorch_dtype[output1_dtype])
out_tensor_1 = pb_utils.Tensor.from_dlpack(
"OUTPUT1", to_dlpack(out_1))
else:
in_0_pytorch, in_1_pytorch = from_dlpack(
in_0.to_dlpack()).cuda(), from_dlpack(
in_1.to_dlpack()).cuda()
out_0, out_1 = (in_0_pytorch - in_1_pytorch,
in_0_pytorch + in_1_pytorch)
out_tensor_0 = pb_utils.Tensor.from_dlpack(
"OUTPUT0", to_dlpack(out_0))
out_tensor_1 = pb_utils.Tensor.from_dlpack(
"OUTPUT1", to_dlpack(out_1))
responses.append(
pb_utils.InferenceResponse([out_tensor_0, out_tensor_1]))
return responses
def execute(self, requests):
responses = []
for request in requests:
input_tensor = pb_utils.get_input_tensor_by_name(request, "IN")
out_tensor = pb_utils.Tensor("OUT", input_tensor.as_numpy())
responses.append(pb_utils.InferenceResponse([out_tensor], error))
return responses
def response_thread(self, response_sender, input0, gpu_output):
# Sleep 5 seconds to make sure the main thread has exited.
time.sleep(5)
if input0.is_cpu():
if not gpu_output[0]:
output0 = pb_utils.Tensor.from_dlpack("OUTPUT0",
input0.to_dlpack())
else:
outptu0_pytorch = from_dlpack(input0.to_dlpack()).cuda()
output0 = pb_utils.Tensor.from_dlpack(
"OUTPUT0", to_dlpack(outptu0_pytorch))
else:
if gpu_output[0]:
output0 = pb_utils.Tensor.from_dlpack("OUTPUT0",
input0.to_dlpack())
else:
output0_pytorch = from_dlpack(input0.to_dlpack()).cpu()
output0 = pb_utils.Tensor.from_dlpack(
"OUTPUT0", to_dlpack(output0_pytorch))
next_gpu_output = pb_utils.Tensor("NEXT_GPU_OUTPUT", gpu_output[1:])
infer_response = pb_utils.InferenceResponse([output0, next_gpu_output])
# Number of times to repeat the response
response_repeat = 2
for _ in range(response_repeat):
response_sender.send(infer_response)
response_sender.send(
flags=pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL)
with self.inflight_thread_count_lck:
self.inflight_thread_count -= 1
def execute(self, requests):
""" This function is called on inference request.
"""
output0_dtype = self.output0_dtype
output1_dtype = self.output1_dtype
responses = []
for request in requests:
input_tensors = request.inputs()
in_0 = pb_utils.get_input_tensor_by_name(request, "INPUT0")
in_1 = pb_utils.get_input_tensor_by_name(request, "INPUT1")
if in_0.as_numpy().dtype.type is np.bytes_ or in_0.as_numpy(
).dtype == np.object:
out_0, out_1 = (in_0.as_numpy().astype(np.int32) - in_1.as_numpy().astype(np.int32),\
in_0.as_numpy().astype(np.int32) + in_1.as_numpy().astype(np.int32))
else:
out_0, out_1 = (in_0.as_numpy() - in_1.as_numpy(),
in_0.as_numpy() + in_1.as_numpy())
out_tensor_0 = pb_utils.Tensor("OUTPUT0",
out_0.astype(output0_dtype))
out_tensor_1 = pb_utils.Tensor("OUTPUT1",
out_1.astype(output1_dtype))
responses.append(
pb_utils.InferenceResponse([out_tensor_0, out_tensor_1]))
return responses
def execute(self, requests):
"""`execute` MUST be implemented in every Python model. `execute`
function receives a list of pb_utils.InferenceRequest as the only
argument. This function is called when an inference request is made
for this model. Depending on the batching configuration (e.g. Dynamic
Batching) used, `requests` may contain multiple requests. Every
Python model, must create one pb_utils.InferenceResponse for every
pb_utils.InferenceRequest in `requests`. If there is an error, you can
set the error argument when creating a pb_utils.InferenceResponse
Parameters
----------
requests : list
A list of pb_utils.InferenceRequest
Returns
-------
list
A list of pb_utils.InferenceResponse. The length of this list must
be the same as `requests`
"""
output0_dtype = self.output0_dtype
output1_dtype = self.output1_dtype
responses = []
# Every Python backend must iterate over everyone of the requests
# and create a pb_utils.InferenceResponse for each of them.
for request in requests:
# Get INPUT0
in_0 = pb_utils.get_input_tensor_by_name(request, "INPUT0")
# Get INPUT1
in_1 = pb_utils.get_input_tensor_by_name(request, "INPUT1")
out_0, out_1 = (in_0.as_numpy() + in_1.as_numpy(),
in_0.as_numpy() - in_1.as_numpy())
# Create output tensors. You need pb_utils.Tensor
# objects to create pb_utils.InferenceResponse.
out_tensor_0 = pb_utils.Tensor("OUTPUT0",
out_0.astype(output0_dtype))
out_tensor_1 = pb_utils.Tensor("OUTPUT1",
out_1.astype(output1_dtype))
# Create InferenceResponse. You can set an error here in case
# there was a problem with handling this inference request.
# Below is an example of how you can set errors in inference
# response:
#
# pb_utils.InferenceResponse(
# output_tensors=..., TritonError("An error occured"))
inference_response = pb_utils.InferenceResponse(
output_tensors=[out_tensor_0, out_tensor_1])
responses.append(inference_response)
# You should return a list of pb_utils.InferenceResponse. Length
# of this list must match the length of `requests` list.
return responses
def execute(self, requests):
"""`execute` must be implemented in every Python model. `execute`
function receives a list of pb_utils.InferenceRequest as the only
argument. This function is called when an inference is requested
for this model. Depending on the batching configuration (e.g. Dynamic
Batching) used, `requests` may contain multiple requests. Every
Python model, must create one pb_utils.InferenceResponse for every
pb_utils.InferenceRequest in `requests`. If there is an error, you can
set the error argument when creating a pb_utils.InferenceResponse.
Parameters
----------
requests : list
A list of pb_utils.InferenceRequest
Returns
-------
list
A list of pb_utils.InferenceResponse. The length of this list must
be the same as `requests`
"""
responses = []
# Every Python backend must iterate over everyone of the requests
# and create a pb_utils.InferenceResponse for each of them.
for request in requests:
# Get INPUT0
input_ids = pb_utils.get_input_tensor_by_name(
request, "input_ids").to_dlpack()
attention_mask = pb_utils.get_input_tensor_by_name(
request, "attention_mask").to_dlpack()
# TODO: Set environment variable to prevent to(self.device)
input_ids = from_dlpack(input_ids).long().to(self.device)
attention_mask = from_dlpack(attention_mask).long().to(self.device)
with torch.no_grad():
outputs = self.model(input_ids, attention_mask)
conf, preds = torch.max(outputs, dim=1)
preds = preds.int()
out_tensor_0 = pb_utils.Tensor("preds", preds.cpu().numpy())
# Create InferenceResponse. You can set an error here in case
# there was a problem with handling this inference request.
# Below is an example of how you can set errors in inference
# response:
#
# pb_utils.InferenceResponse(
# output_tensors=..., TritonError("An error occured"))
inference_response = pb_utils.InferenceResponse(
output_tensors=[out_tensor_0])
responses.append(inference_response)
# You should return a list of pb_utils.InferenceResponse. Length
# of this list must match the length of `requests` list.
return responses
def execute(self, requests):
""" This function is called on inference request.
"""
responses = []
for request in requests:
input_tensor = pb_utils.get_input_tensor_by_name(request, "INPUT0")
out_tensor = pb_utils.Tensor("OUTPUT0", input_tensor.as_numpy())
responses.append(pb_utils.InferenceResponse([out_tensor]))
return responses
def execute(self, requests):
responses = []
for _ in requests:
out_tensor_0 = pb_utils.Tensor(
"OUTPUT0", np.array(['123456'],
dtype=self._dtypes[self._index]))
self._index += 1
responses.append(pb_utils.InferenceResponse([out_tensor_0]))
return responses
def response_thread(self, response_sender, in_input):
# The response_sender is used to send response(s) associated with the
# corresponding request.
# Sleep 5 seconds to make sure the main thread has exited.
time.sleep(5)
status = self.execute_gpu_bls()
if not status:
infer_response = pb_utils.InferenceResponse(
error="GPU BLS test failed.")
response_sender.send(infer_response)
else:
in_value = in_input
infer_request = pb_utils.InferenceRequest(
model_name='identity_fp32',
requested_output_names=["OUTPUT0"],
inputs=[pb_utils.Tensor('INPUT0', in_input)])
infer_response = infer_request.exec()
output0 = pb_utils.get_output_tensor_by_name(
infer_response, "OUTPUT0")
if infer_response.has_error():
response = pb_utils.InferenceResponse(
error=infer_response.error().message())
response_sender.send(
response,
flags=pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL)
elif np.any(in_input != output0.as_numpy()):
error_message = (
"BLS Request input and BLS response output do not match."
f" {in_value} != {output0.as_numpy()}")
response = pb_utils.InferenceResponse(error=error_message)
response_sender.send(
response,
flags=pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL)
else:
output_tensors = [pb_utils.Tensor('OUT', in_value)]
response = pb_utils.InferenceResponse(
output_tensors=output_tensors)
response_sender.send(
response,
flags=pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL)
with self.inflight_thread_count_lck:
self.inflight_thread_count -= 1
def execute(self, requests):
responses = []
for request in requests:
in_0 = pb_utils.get_input_tensor_by_name(request, "INPUT0")
out_tensor_0 = pb_utils.Tensor(
"OUTPUT0",
in_0.as_numpy().astype(self._dtypes[self._index]))
self._index += 1
responses.append(pb_utils.InferenceResponse([out_tensor_0]))
return responses
def execute(self, requests):
"""
Identity model in Python backend.
"""
responses = []
for request in requests:
input_tensor = pb_utils.get_input_tensor_by_name(request, "INPUT0")
out_tensor = pb_utils.Tensor("OUTPUT0", input_tensor.as_numpy())
responses.append(pb_utils.InferenceResponse([out_tensor]))
return responses
def execute(self, requests):
""" This function is called on inference request.
"""
responses = []
for request in requests:
input_tensor = pb_utils.get_input_tensor_by_name(request, "IN")
out_tensor = pb_utils.Tensor("OUT", input_tensor.as_numpy())
error = pb_utils.TritonError('An error occured during execution')
responses.append(pb_utils.InferenceResponse([out_tensor], error))
return responses
def execute(self, requests):
output0_dtype = self.output0_dtype
responses = []
for request in requests:
acc_x = pb_utils.get_input_tensor_by_name(request,
"ACC_X").as_numpy()
acc_y = pb_utils.get_input_tensor_by_name(request,
"ACC_Y").as_numpy()
acc_z = pb_utils.get_input_tensor_by_name(request,
"ACC_Z").as_numpy()
gyro_x = pb_utils.get_input_tensor_by_name(request,
"GYRO_X").as_numpy()
gyro_y = pb_utils.get_input_tensor_by_name(request,
"GYRO_Y").as_numpy()
gyro_z = pb_utils.get_input_tensor_by_name(request,
"GYRO_Z").as_numpy()
humidity = pb_utils.get_input_tensor_by_name(
request, "HUMIDITY").as_numpy()
pressure = pb_utils.get_input_tensor_by_name(
request, "PRESSURE").as_numpy()
temp_hum = pb_utils.get_input_tensor_by_name(
request, "TEMP_HUM").as_numpy()
temp_press = pb_utils.get_input_tensor_by_name(
request, "TEMP_PRESS").as_numpy()
out_0 = np.array([
acc_y, acc_x, acc_z, pressure, temp_press, temp_hum, humidity,
gyro_x, gyro_y, gyro_z
]).transpose()
# ACC_Y ACC_X ACC_Z PRESSURE TEMP_PRESS TEMP_HUM HUMIDITY GYRO_X GYRO_Y GYRO_Z
min = np.array([
-0.132551, -0.049693, 0.759847, 976.001709, 38.724998,
40.220890, 13.003981, -1.937896, -0.265019, -0.250647
])
max = np.array([
0.093099, 0.150289, 1.177543, 1007.996338, 46.093750,
48.355824, 23.506138, 1.923712, 0.219204, 0.671759
])
# MinMax scaling
out_0_scaled = (out_0 - min) / (max - min)
# Create output tensor
out_tensor_0 = pb_utils.Tensor("INPUT0",
out_0_scaled.astype(output0_dtype))
inference_response = pb_utils.InferenceResponse(
output_tensors=[out_tensor_0])
responses.append(inference_response)
return responses
def execute(self, requests):
""" This function is called on inference request.
"""
responses = []
# Only generate the error for the first request
i = 0
for request in requests:
input_tensor = pb_utils.get_input_tensor_by_name(request, "IN")
out_tensor = pb_utils.Tensor("OUT", input_tensor.as_numpy())
if i == 0:
error = pb_utils.TritonError(
'An error occured during execution')
responses.append(
pb_utils.InferenceResponse([out_tensor], error))
else:
responses.append(pb_utils.InferenceResponse([out_tensor]))
i += 1
return responses
def execute(self, requests):
"""Identity model in Python backend that works with GPU and CPU
tensors."""
responses = []
for request in requests:
input_tensor = pb_utils.get_input_tensor_by_name(request, "INPUT0")
out_tensor = pb_utils.Tensor.from_dlpack("OUTPUT0",
input_tensor.to_dlpack())
responses.append(pb_utils.InferenceResponse([out_tensor]))
return responses
async def execute(self, requests):
responses = []
for _ in requests:
# Run the unittest and store the results in InferenceResponse.
result = await test_bls_out_of_memory()
responses.append(
pb_utils.InferenceResponse([
pb_utils.Tensor('OUTPUT0',
np.array([result], dtype=np.float16))
]))
return responses
def execute(self, requests):
"""
The body of this model doesn't matter. The main purpose of this model is
to test correct handling of Python errors in the `finalize` function.
"""
responses = []
for request in requests:
input_tensor = pb_utils.get_input_tensor_by_name(request, "IN")
out_tensor = pb_utils.Tensor("OUT", input_tensor.as_numpy())
responses.append(pb_utils.InferenceResponse([out_tensor], error))
return responses
def execute(self, requests):
responses = []
for request in requests:
input_tensor = pb_utils.get_input_tensor_by_name(request, "IN")
# This tensor is read-only, we need to make a copy
input_data_ro = input_tensor.as_numpy()
input_data = np.array(input_data_ro)
result = self.model(torch.tensor(input_data))
out_tensor = pb_utils.Tensor("OUT", result.detach().numpy())
responses.append(pb_utils.InferenceResponse([out_tensor]))
return responses
def execute(self, requests):
responses = []
for _ in requests:
responses.append(
pb_utils.InferenceResponse(
output_tensors=[],
error=pb_utils.TritonError("An Error Occurred")))
# You must return a list of pb_utils.InferenceResponse. Length
# of this list must match the length of `requests` list.
return responses
def execute(self, requests):
"""
The main purpose of this function is to check whether undefined
variables are correctly handled in `initialize` function. The body of
this function is never called or used.
"""
responses = []
for request in requests:
input_tensor = pb_utils.get_input_tensor_by_name(request, "IN")
out_tensor = pb_utils.Tensor("OUT", input_tensor.as_numpy())
responses.append(pb_utils.InferenceResponse([out_tensor], error))
return responses
def execute(self, requests):
"""
This model ensures that errors in the execute function are properly
handles.
"""
responses = []
for request in requests:
input_tensor = pb_utils.get_input_tensor_by_name(request, "IN")
out_tensor = pb_utils.Tensor("OUT", input_tensor.as_numpy())
lorem_ipsum
responses.append(pb_utils.InferenceResponse([out_tensor]))
return responses
def execute(self, requests):
responses = []
for _ in requests:
# Run the unittest and store the results in InferenceResponse.
test = unittest.main('model', exit=False)
responses.append(
pb_utils.InferenceResponse([
pb_utils.Tensor(
'OUTPUT0',
np.array([test.result.wasSuccessful()],
dtype=np.float16))
]))
return responses
