async def test_predict(model: TempoModel, inference_request: InferenceRequest,
inference_pipeline: Pipeline):
res = await model.predict(inference_request)
assert len(res.outputs) == 1
pipeline_input = to_ndarray(inference_request.inputs[0])
expected_output = inference_pipeline(pipeline_input)
pipeline_output = res.outputs[0].data
assert expected_output.tolist() == pipeline_output
async def test_load_wrapped_class(inference_pipeline_class, inference_request: InferenceRequest):
pipeline_input = to_ndarray(inference_request.inputs[0])
inference_pipeline_class(pipeline_input)
assert inference_pipeline_class.counter == 1
model_settings = case_wrapped_class(inference_pipeline_class)
runtime = InferenceRuntime(model_settings)
await runtime.load()
assert inference_pipeline_class.counter == 1
assert runtime._model._user_func.__self__.counter == 0 # type: ignore
async def test_predict(mlserver_runtime: InferenceRuntime,
inference_request: InferenceRequest):
# NOTE: pytest-cases doesn't wait for async fixtures
# TODO: Raise issue in pytest-cases repo
mlserver_runtime = await mlserver_runtime
res = await mlserver_runtime.predict(inference_request)
assert len(res.outputs) == 1
pipeline_input = to_ndarray(inference_request.inputs[0])
custom_model = copy.copy(mlserver_runtime._model)
# Ensure direct call to class does not try to do remote
custom_model.set_remote(False)
expected_output = custom_model(payload=pipeline_input)
pipeline_output = res.outputs[0].data
assert expected_output.tolist() == pipeline_output
def _check_request(
self, payload: types.InferenceRequest) -> types.InferenceRequest:
if len(payload.inputs) != 1:
raise InferenceError(
"LightGBM only supports a single input tensor "
f"({len(payload.inputs)} were received)")
# Convert to `numpy.ndarray` and store in parameters
try:
model_input = payload.inputs[0]
array_data = to_ndarray(model_input)
model_input.parameters = {"data": array_data}
except Exception as e:
# There are a few things that can go wrong here, e.g. less than 2-D
# in the array), or input data not compatible with a numpy array
raise InferenceError("Invalid input to LightGBM") from e
return payload
def _predict_outputs(
self,
payload: types.InferenceRequest) -> List[types.ResponseOutput]:
model_input = payload.inputs[0]
input_data = to_ndarray(model_input)
outputs = []
for request_output in payload.outputs: # type: ignore
predict_fn = getattr(self._model, request_output.name)
y = predict_fn(input_data)
# TODO: Set datatype (cast from numpy?)
outputs.append(
types.ResponseOutput(
name=request_output.name,
shape=y.shape,
datatype="FP32",
data=y.tolist(),
))
return outputs
def _check_request(
self, payload: types.InferenceRequest) -> types.InferenceRequest:
if len(payload.inputs) != 1:
raise InferenceError(
"XGBoostModel only supports a single input tensor "
f"({len(payload.inputs)} were received)")
# Convert to `xgboost.DMatrix` and store in parameters
# TODO: Move this out to "types conversion" pipeline, once it's there.
try:
model_input = payload.inputs[0]
array_data = to_ndarray(model_input)
dmatrix_data = xgb.DMatrix(array_data)
# TODO: Use Parameters object
model_input.parameters = {
"dmatrix_data": dmatrix_data
} # type: ignore
except Exception as e:
# There are a few things that can go wrong here, e.g. less than 2-D
# in the array), or input data not compatible with a numpy array
raise InferenceError("Invalid input to XGBoostModel") from e
return payload