def test_create_rest_reponse_ndarray():
user_model = UserObject()
request = {"data": {"ndarray": np.array([[5, 6, 7]]), "names": []}}
raw_response = np.array([[1, 2, 3]])
result = scu.construct_response_json(user_model, True, request, raw_response)
assert "ndarray" in result.get("data", {})
assert np.array_equal(result["data"]["ndarray"], raw_response)
def route(
user_model: Any, request: Union[prediction_pb2.SeldonMessage, List, Dict]
) -> Union[prediction_pb2.SeldonMessage, List, Dict]:
"""
Parameters
----------
user_model
A Seldon user model
request
A SelodonMessage proto
Returns
-------
"""
is_proto = isinstance(request, prediction_pb2.SeldonMessage)
if hasattr(user_model, "route_rest"):
logger.warning("route_rest is deprecated. Please use route_raw")
return user_model.route_rest(request)
elif hasattr(user_model, "route_grpc"):
logger.warning("route_grpc is deprecated. Please use route_raw")
return user_model.route_grpc(request)
else:
if hasattr(user_model, "route_raw"):
try:
return user_model.route_raw(request)
except SeldonNotImplementedError:
pass
if is_proto:
(features, meta, datadef,
data_type) = extract_request_parts(request)
client_response = client_route(user_model,
features,
datadef.names,
meta=meta)
if not isinstance(client_response, int):
raise SeldonMicroserviceException(
"Routing response must be int but got " +
str(client_response))
client_response_arr = np.array([[client_response]])
return construct_response(user_model, False, request,
client_response_arr)
else:
(features, meta, datadef,
data_type) = extract_request_parts_json(request)
class_names = datadef[
"names"] if datadef and "names" in datadef else []
client_response = client_route(user_model,
features,
class_names,
meta=meta)
if not isinstance(client_response, int):
raise SeldonMicroserviceException(
"Routing response must be int but got " +
str(client_response))
client_response_arr = np.array([[client_response]])
return construct_response_json(user_model, False, request,
client_response_arr)
def health_status(
user_model: Any, seldon_metrics: SeldonMetrics
) -> Union[prediction_pb2.SeldonMessage, List, Dict]:
"""
Call the user model to check the health of the model
Parameters
----------
user_model
User defined class instance
seldon_metrics
A SeldonMetrics instance
Returns
-------
Health check output
"""
if hasattr(user_model, "health_status_raw"):
try:
return user_model.health_status_raw()
except SeldonNotImplementedError:
pass
client_response = client_health_status(user_model)
metrics = client_custom_metrics(user_model, seldon_metrics,
HEALTH_METRIC_METHOD_TAG)
return construct_response_json(user_model, False, {}, client_response,
None, metrics)
def test_create_rest_reponse_nparray():
user_model = UserObject()
request = {}
raw_response = np.array([[1, 2, 3]])
result = scu.construct_response_json(user_model, True, request, raw_response)
assert "tensor" in result.get("data", {})
assert result["data"]["tensor"]["values"] == [1, 2, 3]
def test_create_rest_reponse_tensor():
user_model = UserObject()
tensor = {"values": [1, 2, 3], "shape": (3,)}
request = {"data": {"tensor": tensor, "names": []}}
raw_response = np.array([1, 2, 3])
result = scu.construct_response_json(user_model, True, request, raw_response)
assert "tensor" in result.get("data", {})
assert np.array_equal(result["data"]["tensor"], tensor)
def test_create_rest_reponse_binary():
user_model = UserObject()
request_data = b"input"
request = {"binData": request_data}
raw_response = b"binary"
sm = scu.construct_response_json(user_model, True, request, raw_response)
assert "strData" not in sm
assert "binData" in sm
assert sm["binData"] == raw_response
def test_create_rest_response_strdata():
user_model = UserObject()
request_data = "Request data"
request = {"strData": request_data}
raw_response = "hello world"
sm = scu.construct_response_json(user_model, True, request, raw_response)
assert "strData" in sm
assert len(sm["strData"]) > 0
assert sm["strData"] == raw_response
def transform_output(
user_model: Any,
request: Union[prediction_pb2.SeldonMessage, List, Dict]) \
-> Union[prediction_pb2.SeldonMessage, List, Dict]:
"""
Parameters
----------
user_model
User defined class to handle transform input
request
The incoming request
Returns
-------
The transformed request
"""
is_proto = isinstance(request, prediction_pb2.SeldonMessage)
if hasattr(user_model, "transform_output_rest"):
logger.warning(
"transform_input_rest is deprecated. Please use transform_input_raw"
)
return user_model.transform_output_rest(request)
elif hasattr(user_model, "transform_output_grpc"):
logger.warning(
"transform_input_grpc is deprecated. Please use transform_input_raw"
)
return user_model.transform_output_grpc(request)
else:
if hasattr(user_model, "transform_output_raw"):
try:
return user_model.transform_output_raw(request)
except SeldonNotImplementedError:
pass
if is_proto:
(features, meta, datadef,
data_type) = extract_request_parts(request)
client_response = client_transform_output(user_model,
features,
datadef.names,
meta=meta)
return construct_response(user_model, False, request,
client_response)
else:
(features, meta, datadef,
data_type) = extract_request_parts_json(request)
class_names = datadef[
"names"] if datadef and "names" in datadef else []
client_response = client_transform_output(user_model,
features,
class_names,
meta=meta)
return construct_response_json(user_model, False, request,
client_response)
def test_create_rest_response_binary():
user_model = UserObject()
request_data = b"input"
request = {"binData": request_data}
raw_resp = b"binary"
sm = scu.construct_response_json(user_model, True, request, raw_resp)
resp_data = base64.b64encode(raw_resp).decode("utf-8")
assert "strData" not in sm
assert "binData" in sm
assert sm["binData"] == resp_data
def test_create_rest_response_jsondata_with_array_input():
user_model = UserObject(ret_dict=True)
request_data = np.array([[5, 6, 7]])
datadef = scu.array_to_rest_datadef("ndarray", request_data)
json_request = {"data": datadef}
raw_response = {"output": "data"}
json_response = scu.construct_response_json(user_model, True, json_request,
raw_response)
assert "data" not in json_response
assert json_response["jsonData"] == user_model.dict
def test_create_rest_response_jsondata():
user_model = UserObject()
request_data = np.array([[5, 6, 7]])
datadef = scu.array_to_rest_datadef("ndarray", request_data)
json_request = { "jsonData": datadef }
raw_response = {"output": "data"}
json_response = scu.construct_response_json(user_model, True, json_request, raw_response)
assert "data" not in json_response
emptyValue = Value()
assert json_response["jsonData"] != emptyValue
def test_create_rest_reponse_text_ndarray():
user_model = UserObject()
request_data = np.array([["hello", "world"], ["hello", "another", "world"]])
request = {"data": {"ndarray": request_data, "names": []}}
(features, meta, datadef, data_type) = scu.extract_request_parts_json(request)
raw_response = np.array([["hello", "world"], ["here", "another"]])
result = scu.construct_response_json(user_model, True, request, raw_response)
assert "ndarray" in result.get("data", {})
assert np.array_equal(result["data"]["ndarray"], raw_response)
assert datadef == request["data"]
assert np.array_equal(features, request_data)
assert data_type == "data"
def predict(
user_model: Any, request: Union[prediction_pb2.SeldonMessage, List, Dict]
) -> Union[prediction_pb2.SeldonMessage, List, Dict]:
"""
Call the user model to get a prediction and package the response
Parameters
----------
user_model
User defined class instance
request
The incoming request
Returns
-------
The prediction
"""
is_proto = isinstance(request, prediction_pb2.SeldonMessage)
if hasattr(user_model, "predict_rest") and not is_proto:
logger.warning("predict_rest is deprecated. Please use predict_raw")
return user_model.predict_rest(request)
elif hasattr(user_model, "predict_grpc") and is_proto:
logger.warning("predict_grpc is deprecated. Please use predict_raw")
return user_model.predict_grpc(request)
else:
if hasattr(user_model, "predict_raw"):
try:
return user_model.predict_raw(request)
except SeldonNotImplementedError:
pass
if is_proto:
(features, meta, datadef,
data_type) = extract_request_parts(request)
client_response = client_predict(user_model,
features,
datadef.names,
meta=meta)
return construct_response(user_model, False, request,
client_response)
else:
(features, meta, datadef,
data_type) = extract_request_parts_json(request)
class_names = datadef[
"names"] if datadef and "names" in datadef else []
client_response = client_predict(user_model,
features,
class_names,
meta=meta)
return construct_response_json(user_model, False, request,
client_response)
def health_status(user_model: Any) -> Union[prediction_pb2.SeldonMessage, List, Dict]:
"""
Call the user model to check the health of the model
Parameters
----------
user_model
User defined class instance
Returns
-------
Health check output
"""
if hasattr(user_model, "health_status_raw"):
try:
return user_model.health_status_raw()
except SeldonNotImplementedError:
pass
client_response = client_health_status(user_model)
return construct_response_json(user_model, False, {}, client_response)
def aggregate(
user_model: Any,
request: Union[prediction_pb2.SeldonMessageList, List, Dict],
seldon_metrics: SeldonMetrics,
) -> Union[prediction_pb2.SeldonMessage, List, Dict]:
"""
Aggregate a list of payloads
Parameters
----------
user_model
A Seldon user model
request
SeldonMessage proto
seldon_metrics
A SeldonMetrics instance
Returns
-------
Aggregated SeldonMessage proto
"""
def merge_meta(meta_list):
tags = {}
for meta in meta_list:
if meta:
tags.update(meta.get("tags", {}))
return {"tags": tags}
def merge_metrics(meta_list, custom_metrics):
metrics = []
for meta in meta_list:
if meta:
metrics.extend(meta.get("metrics", []))
metrics.extend(custom_metrics)
return metrics
is_proto = isinstance(request, prediction_pb2.SeldonMessageList)
if hasattr(user_model, "aggregate_rest"):
logger.warning(
"aggregate_rest is deprecated. Please use aggregate_raw")
return user_model.aggregate_rest(request)
elif hasattr(user_model, "aggregate_grpc"):
logger.warning(
"aggregate_grpc is deprecated. Please use aggregate_raw")
return user_model.aggregate_grpc(request)
else:
if hasattr(user_model, "aggregate_raw"):
try:
response = user_model.aggregate_raw(request)
handle_raw_custom_metrics(response, seldon_metrics, is_proto,
AGGREGATE_METRIC_METHOD_TAG)
return response
except SeldonNotImplementedError:
pass
if is_proto:
features_list = []
names_list = []
meta_list = []
for msg in request.seldonMessages:
(features, meta, datadef,
data_type) = extract_request_parts(msg)
features_list.append(features)
names_list.append(datadef.names)
meta_list.append(meta)
client_response = client_aggregate(user_model, features_list,
names_list)
metrics = client_custom_metrics(
user_model,
seldon_metrics,
AGGREGATE_METRIC_METHOD_TAG,
client_response.metrics,
)
return construct_response(
user_model,
False,
request.seldonMessages[0],
client_response.data,
merge_meta(meta_list),
merge_metrics(meta_list, metrics),
client_response.tags,
)
else:
features_list = []
names_list = []
if isinstance(request, list):
msgs = request
elif "seldonMessages" in request and isinstance(
request["seldonMessages"], list):
msgs = request["seldonMessages"]
else:
raise SeldonMicroserviceException(
f"Invalid request data type: {request}")
meta_list = []
for msg in msgs:
(features, meta, datadef,
data_type) = extract_request_parts_json(msg)
class_names = datadef[
"names"] if datadef and "names" in datadef else []
features_list.append(features)
names_list.append(class_names)
meta_list.append(meta)
client_response = client_aggregate(user_model, features_list,
names_list)
metrics = client_custom_metrics(
user_model,
seldon_metrics,
AGGREGATE_METRIC_METHOD_TAG,
client_response.metrics,
)
return construct_response_json(
user_model,
False,
msgs[0],
client_response.data,
merge_meta(meta_list),
merge_metrics(meta_list, metrics),
client_response.tags,
)
def predict(
user_model: Any,
request: Union[prediction_pb2.SeldonMessage, List, Dict, bytes],
seldon_metrics: SeldonMetrics,
) -> Union[prediction_pb2.SeldonMessage, List, Dict, bytes]:
"""
Call the user model to get a prediction and package the response
Parameters
----------
user_model
User defined class instance
request
The incoming request
Returns
-------
The prediction
"""
# TODO: Find a way to choose predict_rest or predict_grpc when payload is
# not decoded
is_proto = isinstance(request, prediction_pb2.SeldonMessage)
if hasattr(user_model, "predict_rest") and not is_proto:
logger.warning("predict_rest is deprecated. Please use predict_raw")
return user_model.predict_rest(request)
elif hasattr(user_model, "predict_grpc") and is_proto:
logger.warning("predict_grpc is deprecated. Please use predict_raw")
return user_model.predict_grpc(request)
else:
if hasattr(user_model, "predict_raw"):
try:
response = user_model.predict_raw(request)
handle_raw_custom_metrics(response, seldon_metrics, is_proto,
PREDICT_METRIC_METHOD_TAG)
return response
except SeldonNotImplementedError:
pass
if is_proto:
(features, meta, datadef,
data_type) = extract_request_parts(request)
client_response = client_predict(user_model,
features,
datadef.names,
meta=meta)
metrics = client_custom_metrics(
user_model,
seldon_metrics,
PREDICT_METRIC_METHOD_TAG,
client_response.metrics,
)
return construct_response(
user_model,
False,
request,
client_response.data,
meta,
metrics,
client_response.tags,
)
else:
(features, meta, datadef,
data_type) = extract_request_parts_json(request)
class_names = datadef[
"names"] if datadef and "names" in datadef else []
client_response = client_predict(user_model,
features,
class_names,
meta=meta)
metrics = client_custom_metrics(
user_model,
seldon_metrics,
PREDICT_METRIC_METHOD_TAG,
client_response.metrics,
)
return construct_response_json(
user_model,
False,
request,
client_response.data,
meta,
metrics,
client_response.tags,
)
def aggregate(
user_model: Any, request: Union[prediction_pb2.SeldonMessageList, List,
Dict]
) -> Union[prediction_pb2.SeldonMessage, List, Dict]:
"""
Aggregate a list of payloads
Parameters
----------
user_model
A Seldon user model
request
SeldonMessage proto
Returns
-------
Aggregated SeldonMessage proto
"""
is_proto = isinstance(request, prediction_pb2.SeldonMessageList)
if hasattr(user_model, "aggregate_rest"):
logger.warning(
"aggregate_rest is deprecated. Please use aggregate_raw")
return user_model.aggregate_rest(request)
elif hasattr(user_model, "aggregate_grpc"):
logger.warning(
"aggregate_grpc is deprecated. Please use aggregate_raw")
return user_model.aggregate_grpc(request)
else:
if hasattr(user_model, "aggregate_raw"):
try:
return user_model.aggregate_raw(request)
except SeldonNotImplementedError:
pass
if is_proto:
features_list = []
names_list = []
for msg in request.seldonMessages:
(features, meta, datadef,
data_type) = extract_request_parts(msg)
features_list.append(features)
names_list.append(datadef.names)
client_response = client_aggregate(user_model, features_list,
names_list)
return construct_response(user_model, False,
request.seldonMessages[0],
client_response)
else:
features_list = []
names_list = []
if isinstance(request, list):
msgs = request
elif "seldonMessages" in request and isinstance(
request["seldonMessages"], list):
msgs = request["seldonMessages"]
else:
raise SeldonMicroserviceException(
f"Invalid request data type: {request}")
for msg in msgs:
(features, meta, datadef,
data_type) = extract_request_parts_json(msg)
class_names = datadef[
"names"] if datadef and "names" in datadef else []
features_list.append(features)
names_list.append(class_names)
client_response = client_aggregate(user_model, features_list,
names_list)
return construct_response_json(user_model, False, msgs[0],
client_response)
def transform_input(
user_model: Any,
request: Union[prediction_pb2.SeldonMessage, List, Dict],
seldon_metrics: SeldonMetrics,
) -> Union[prediction_pb2.SeldonMessage, List, Dict]:
"""
Parameters
----------
user_model
User defined class to handle transform input
request
The incoming request
Returns
-------
The transformed request
"""
is_proto = isinstance(request, prediction_pb2.SeldonMessage)
if hasattr(user_model, "transform_input_rest"):
logger.warning(
"transform_input_rest is deprecated. Please use transform_input_raw"
)
return user_model.transform_input_rest(request)
elif hasattr(user_model, "transform_input_grpc"):
logger.warning(
"transform_input_grpc is deprecated. Please use transform_input_raw"
)
return user_model.transform_input_grpc(request)
else:
if hasattr(user_model, "transform_input_raw"):
try:
response = user_model.transform_input_raw(request)
if is_proto:
metrics = seldon_message_to_json(response.meta).get(
"metrics", [])
else:
metrics = response.get("meta", {}).get("metrics", [])
seldon_metrics.update(metrics)
return response
except SeldonNotImplementedError:
pass
if is_proto:
(features, meta, datadef,
data_type) = extract_request_parts(request)
client_response = client_transform_input(user_model,
features,
datadef.names,
meta=meta)
metrics = client_custom_metrics(user_model)
if seldon_metrics is not None:
seldon_metrics.update(metrics)
return construct_response(user_model, False, request,
client_response, meta, metrics)
else:
(features, meta, datadef,
data_type) = extract_request_parts_json(request)
class_names = datadef[
"names"] if datadef and "names" in datadef else []
client_response = client_transform_input(user_model,
features,
class_names,
meta=meta)
metrics = client_custom_metrics(user_model)
if seldon_metrics is not None:
seldon_metrics.update(metrics)
return construct_response_json(user_model, False, request,
client_response, meta, metrics)
