async def predict(self, payload: types.InferenceRequest) -> types.InferenceResponse:
total = 0
for inp in payload.inputs:
total += sum(inp.data)
output = types.ResponseOutput(
name="total", shape=[1], datatype="FP32", data=[total]
)
return types.InferenceResponse(model_name=self.name, id="1", outputs=[output])
async def predict(self, payload: types.InferenceRequest) -> types.InferenceResponse:
payload = self._check_request(payload)
prediction = self._model.predict(payload.inputs[0].data)
return types.InferenceResponse(
model_name=self.name,
model_version=self.version,
outputs=[
types.ResponseOutput(
name="predict",
shape=[1],
datatype="FP32",
data=prediction,
)
],
)
def predict(self,
payload: types.InferenceRequest) -> types.InferenceResponse:
inputs = self._extract_inputs(payload)
predictions = self._predictive(rng_key=random.PRNGKey(0), **inputs)
obs = predictions['obs']
obs_mean = obs.mean()
return types.InferenceResponse(
id=payload.id,
model_name=self.name,
model_version=self.version,
outputs=[
types.ResponseOutput(name="obs_mean",
shape=obs_mean.shape,
datatype="FP32",
data=np.asarray(obs_mean).tolist())
])
def _predict_outputs(
self,
payload: types.InferenceRequest) -> List[types.ResponseOutput]:
model_input = payload.inputs[0]
input_data = to_ndarray(model_input)
outputs = []
for request_output in payload.outputs: # type: ignore
predict_fn = getattr(self._model, request_output.name)
y = predict_fn(input_data)
# TODO: Set datatype (cast from numpy?)
outputs.append(
types.ResponseOutput(
name=request_output.name,
shape=y.shape,
datatype="FP32",
data=y.tolist(),
))
return outputs
async def predict(self, payload: types.InferenceRequest) -> types.InferenceResponse:
payload = self._check_request(payload)
# _check_request will convert the data to `np.ndarray`
model_input = payload.inputs[0]
data = model_input.parameters["data"] # type: ignore
prediction = self._model.predict(data)
# TODO: Set datatype (cast from numpy?)
return types.InferenceResponse(
model_name=self.name,
model_version=self.version,
outputs=[
types.ResponseOutput(
name="predict",
shape=prediction.shape,
datatype="FP32",
data=prediction.tolist(),
)
],
)