def health_status(
user_model: Any, seldon_metrics: SeldonMetrics
) -> Union[prediction_pb2.SeldonMessage, List, Dict]:
"""
Call the user model to check the health of the model
Parameters
----------
user_model
User defined class instance
seldon_metrics
A SeldonMetrics instance
Returns
-------
Health check output
"""
if hasattr(user_model, "health_status_raw"):
try:
return user_model.health_status_raw()
except SeldonNotImplementedError:
pass
client_response = client_health_status(user_model)
metrics = client_custom_metrics(user_model, seldon_metrics,
HEALTH_METRIC_METHOD_TAG)
return construct_response_json(user_model, False, {}, client_response,
None, metrics)
def construct_response(user_model: SeldonComponent, is_request: bool, client_request: prediction_pb2.SeldonMessage,
client_raw_response: Union[np.ndarray, str, bytes, dict]) -> prediction_pb2.SeldonMessage:
"""
Parameters
----------
user_model
Client user class
is_request
Whether this is part of the request flow as opposed to the response flow
client_request
The request received
client_raw_response
The raw client response from their model
Returns
-------
A SeldonMessage proto response
"""
data_type = client_request.WhichOneof("data_oneof")
meta = prediction_pb2.Meta()
meta_json: Dict = {}
tags = client_custom_tags(user_model)
if tags:
meta_json["tags"] = tags
metrics = client_custom_metrics(user_model)
if metrics:
meta_json["metrics"] = metrics
if client_request.meta:
if client_request.meta.puid:
meta_json["puid"] = client_request.meta.puid
json_format.ParseDict(meta_json, meta)
if isinstance(client_raw_response, np.ndarray) or isinstance(client_raw_response, list):
client_raw_response = np.array(client_raw_response)
if is_request:
names = client_feature_names(user_model, client_request.data.names)
else:
names = client_class_names(user_model, client_raw_response)
if data_type == "data": # If request is using defaultdata then return what was sent if is numeric response else ndarray
if np.issubdtype(client_raw_response.dtype, np.number):
default_data_type = client_request.data.WhichOneof("data_oneof")
else:
default_data_type = "ndarray"
else: # If numeric response return as tensor else return as ndarray
if np.issubdtype(client_raw_response.dtype, np.number):
default_data_type = "tensor"
else:
default_data_type = "ndarray"
data = array_to_grpc_datadef(default_data_type, client_raw_response, names)
return prediction_pb2.SeldonMessage(data=data, meta=meta)
elif isinstance(client_raw_response, str):
return prediction_pb2.SeldonMessage(strData=client_raw_response, meta=meta)
elif isinstance(client_raw_response, dict):
jsonDataResponse = ParseDict(client_raw_response, prediction_pb2.SeldonMessage().jsonData)
return prediction_pb2.SeldonMessage(jsonData=jsonDataResponse, meta=meta)
elif isinstance(client_raw_response, (bytes, bytearray)):
return prediction_pb2.SeldonMessage(binData=client_raw_response, meta=meta)
else:
raise SeldonMicroserviceException("Unknown data type returned as payload:" + client_raw_response)
def test_component_bad():
with pytest.raises(SeldonMicroserviceException):
c = Component(False)
client_custom_metrics(c, SeldonMetrics(), TEST_METRIC_METHOD_TAG)
def test_component_ok():
c = Component(True)
print(client_custom_metrics(c, SeldonMetrics(), TEST_METRIC_METHOD_TAG))
assert client_custom_metrics(
c, SeldonMetrics(),
TEST_METRIC_METHOD_TAG) == [EXPECTED_COUNTER_METRIC]
def predict(
user_model: Any,
request: Union[prediction_pb2.SeldonMessage, List, Dict, bytes],
seldon_metrics: SeldonMetrics,
) -> Union[prediction_pb2.SeldonMessage, List, Dict, bytes]:
"""
Call the user model to get a prediction and package the response
Parameters
----------
user_model
User defined class instance
request
The incoming request
Returns
-------
The prediction
"""
# TODO: Find a way to choose predict_rest or predict_grpc when payload is
# not decoded
is_proto = isinstance(request, prediction_pb2.SeldonMessage)
if hasattr(user_model, "predict_rest") and not is_proto:
logger.warning("predict_rest is deprecated. Please use predict_raw")
return user_model.predict_rest(request)
elif hasattr(user_model, "predict_grpc") and is_proto:
logger.warning("predict_grpc is deprecated. Please use predict_raw")
return user_model.predict_grpc(request)
else:
if hasattr(user_model, "predict_raw"):
try:
response = user_model.predict_raw(request)
handle_raw_custom_metrics(response, seldon_metrics, is_proto,
PREDICT_METRIC_METHOD_TAG)
return response
except SeldonNotImplementedError:
pass
if is_proto:
(features, meta, datadef,
data_type) = extract_request_parts(request)
client_response = client_predict(user_model,
features,
datadef.names,
meta=meta)
metrics = client_custom_metrics(
user_model,
seldon_metrics,
PREDICT_METRIC_METHOD_TAG,
client_response.metrics,
)
return construct_response(
user_model,
False,
request,
client_response.data,
meta,
metrics,
client_response.tags,
)
else:
(features, meta, datadef,
data_type) = extract_request_parts_json(request)
class_names = datadef[
"names"] if datadef and "names" in datadef else []
client_response = client_predict(user_model,
features,
class_names,
meta=meta)
metrics = client_custom_metrics(
user_model,
seldon_metrics,
PREDICT_METRIC_METHOD_TAG,
client_response.metrics,
)
return construct_response_json(
user_model,
False,
request,
client_response.data,
meta,
metrics,
client_response.tags,
)
def aggregate(
user_model: Any,
request: Union[prediction_pb2.SeldonMessageList, List, Dict],
seldon_metrics: SeldonMetrics,
) -> Union[prediction_pb2.SeldonMessage, List, Dict]:
"""
Aggregate a list of payloads
Parameters
----------
user_model
A Seldon user model
request
SeldonMessage proto
seldon_metrics
A SeldonMetrics instance
Returns
-------
Aggregated SeldonMessage proto
"""
def merge_meta(meta_list):
tags = {}
for meta in meta_list:
if meta:
tags.update(meta.get("tags", {}))
return {"tags": tags}
def merge_metrics(meta_list, custom_metrics):
metrics = []
for meta in meta_list:
if meta:
metrics.extend(meta.get("metrics", []))
metrics.extend(custom_metrics)
return metrics
is_proto = isinstance(request, prediction_pb2.SeldonMessageList)
if hasattr(user_model, "aggregate_rest"):
logger.warning(
"aggregate_rest is deprecated. Please use aggregate_raw")
return user_model.aggregate_rest(request)
elif hasattr(user_model, "aggregate_grpc"):
logger.warning(
"aggregate_grpc is deprecated. Please use aggregate_raw")
return user_model.aggregate_grpc(request)
else:
if hasattr(user_model, "aggregate_raw"):
try:
response = user_model.aggregate_raw(request)
handle_raw_custom_metrics(response, seldon_metrics, is_proto,
AGGREGATE_METRIC_METHOD_TAG)
return response
except SeldonNotImplementedError:
pass
if is_proto:
features_list = []
names_list = []
meta_list = []
for msg in request.seldonMessages:
(features, meta, datadef,
data_type) = extract_request_parts(msg)
features_list.append(features)
names_list.append(datadef.names)
meta_list.append(meta)
client_response = client_aggregate(user_model, features_list,
names_list)
metrics = client_custom_metrics(
user_model,
seldon_metrics,
AGGREGATE_METRIC_METHOD_TAG,
client_response.metrics,
)
return construct_response(
user_model,
False,
request.seldonMessages[0],
client_response.data,
merge_meta(meta_list),
merge_metrics(meta_list, metrics),
client_response.tags,
)
else:
features_list = []
names_list = []
if isinstance(request, list):
msgs = request
elif "seldonMessages" in request and isinstance(
request["seldonMessages"], list):
msgs = request["seldonMessages"]
else:
raise SeldonMicroserviceException(
f"Invalid request data type: {request}")
meta_list = []
for msg in msgs:
(features, meta, datadef,
data_type) = extract_request_parts_json(msg)
class_names = datadef[
"names"] if datadef and "names" in datadef else []
features_list.append(features)
names_list.append(class_names)
meta_list.append(meta)
client_response = client_aggregate(user_model, features_list,
names_list)
metrics = client_custom_metrics(
user_model,
seldon_metrics,
AGGREGATE_METRIC_METHOD_TAG,
client_response.metrics,
)
return construct_response_json(
user_model,
False,
msgs[0],
client_response.data,
merge_meta(meta_list),
merge_metrics(meta_list, metrics),
client_response.tags,
)
def route(
user_model: Any,
request: Union[prediction_pb2.SeldonMessage, List, Dict],
seldon_metrics: SeldonMetrics,
) -> Union[prediction_pb2.SeldonMessage, List, Dict]:
"""
Parameters
----------
user_model
A Seldon user model
request
A SelodonMessage proto
seldon_metrics
A SeldonMetrics instance
Returns
-------
"""
is_proto = isinstance(request, prediction_pb2.SeldonMessage)
if hasattr(user_model, "route_rest"):
logger.warning("route_rest is deprecated. Please use route_raw")
return user_model.route_rest(request)
elif hasattr(user_model, "route_grpc"):
logger.warning("route_grpc is deprecated. Please use route_raw")
return user_model.route_grpc(request)
else:
if hasattr(user_model, "route_raw"):
try:
response = user_model.route_raw(request)
handle_raw_custom_metrics(response, seldon_metrics, is_proto,
ROUTER_METRIC_METHOD_TAG)
return response
except SeldonNotImplementedError:
pass
if is_proto:
(features, meta, datadef,
data_type) = extract_request_parts(request)
client_response = client_route(user_model,
features,
datadef.names,
meta=meta)
if not isinstance(client_response.data, int):
raise SeldonMicroserviceException(
"Routing response must be int but got " +
str(client_response.data))
client_response_arr = np.array([[client_response.data]])
metrics = client_custom_metrics(
user_model,
seldon_metrics,
ROUTER_METRIC_METHOD_TAG,
client_response.metrics,
)
return construct_response(
user_model,
False,
request,
client_response_arr,
None,
metrics,
client_response.tags,
)
else:
(features, meta, datadef,
data_type) = extract_request_parts_json(request)
class_names = datadef[
"names"] if datadef and "names" in datadef else []
client_response = client_route(user_model,
features,
class_names,
meta=meta)
if not isinstance(client_response.data, int):
raise SeldonMicroserviceException(
"Routing response must be int but got " +
str(client_response.data))
client_response_arr = np.array([[client_response.data]])
metrics = client_custom_metrics(
user_model,
seldon_metrics,
ROUTER_METRIC_METHOD_TAG,
client_response.metrics,
)
return construct_response_json(
user_model,
False,
request,
client_response_arr,
None,
metrics,
client_response.tags,
)
def transform_output(
user_model: Any,
request: Union[prediction_pb2.SeldonMessage, List, Dict],
seldon_metrics: SeldonMetrics,
) -> Union[prediction_pb2.SeldonMessage, List, Dict]:
"""
Parameters
----------
user_model
User defined class to handle transform input
request
The incoming request
Returns
-------
The transformed request
"""
is_proto = isinstance(request, prediction_pb2.SeldonMessage)
if hasattr(user_model, "transform_output_rest"):
logger.warning(
"transform_input_rest is deprecated. Please use transform_input_raw"
)
return user_model.transform_output_rest(request)
elif hasattr(user_model, "transform_output_grpc"):
logger.warning(
"transform_input_grpc is deprecated. Please use transform_input_raw"
)
return user_model.transform_output_grpc(request)
else:
if hasattr(user_model, "transform_output_raw"):
try:
response = user_model.transform_output_raw(request)
handle_raw_custom_metrics(
response,
seldon_metrics,
is_proto,
OUTPUT_TRANSFORM_METRIC_METHOD_TAG,
)
return response
except SeldonNotImplementedError:
pass
if is_proto:
(features, meta, datadef,
data_type) = extract_request_parts(request)
client_response = client_transform_output(user_model,
features,
datadef.names,
meta=meta)
metrics = client_custom_metrics(
user_model,
seldon_metrics,
OUTPUT_TRANSFORM_METRIC_METHOD_TAG,
client_response.metrics,
)
return construct_response(
user_model,
False,
request,
client_response.data,
meta,
metrics,
client_response.tags,
)
else:
(features, meta, datadef,
data_type) = extract_request_parts_json(request)
class_names = datadef[
"names"] if datadef and "names" in datadef else []
client_response = client_transform_output(user_model,
features,
class_names,
meta=meta)
metrics = client_custom_metrics(
user_model,
seldon_metrics,
OUTPUT_TRANSFORM_METRIC_METHOD_TAG,
client_response.metrics,
)
return construct_response_json(
user_model,
False,
request,
client_response.data,
meta,
metrics,
client_response.tags,
)
def send_feedback(
user_model: Any,
request: prediction_pb2.Feedback,
predictive_unit_id: str,
seldon_metrics: SeldonMetrics,
) -> prediction_pb2.SeldonMessage:
"""
Parameters
----------
user_model
A Seldon user model
request
SeldonMesage proto
predictive_unit_id
The ID of the enclosing container predictive unit. Will be taken from environment.
Returns
-------
"""
seldon_metrics.update_reward(request.reward)
if hasattr(user_model, "send_feedback_rest"):
logger.warning(
"send_feedback_rest is deprecated. Please use send_feedback_raw")
request_json = json_format.MessageToJson(request)
response_json = user_model.send_feedback_rest(request_json)
return json_to_seldon_message(response_json)
elif hasattr(user_model, "send_feedback_grpc"):
logger.warning(
"send_feedback_grpc is deprecated. Please use send_feedback_raw")
response_json = user_model.send_feedback_grpc(request)
return json_to_seldon_message(response_json)
else:
if hasattr(user_model, "send_feedback_raw"):
try:
response = user_model.send_feedback_raw(request)
handle_raw_custom_metrics(response, seldon_metrics, True,
FEEDBACK_METRIC_METHOD_TAG)
return response
except SeldonNotImplementedError:
pass
(datadef_request, features, truth,
reward) = extract_feedback_request_parts(request)
routing = request.response.meta.routing.get(predictive_unit_id)
client_response = client_send_feedback(user_model, features,
datadef_request.names, reward,
truth, routing)
metrics = client_custom_metrics(
user_model,
seldon_metrics,
FEEDBACK_METRIC_METHOD_TAG,
client_response.metrics,
)
if client_response.data is None:
client_response.data = np.array([])
return construct_response(
user_model,
False,
request.request,
client_response.data,
None,
metrics,
client_response.tags,
)
def construct_response_json(
user_model: SeldonComponent,
is_request: bool,
client_request_raw: Union[List, Dict],
client_raw_response: Union[np.ndarray, str, bytes, dict],
meta: dict = None,
) -> Union[List, Dict]:
"""
This class converts a raw REST response into a JSON object that has the same structure as
the SeldonMessage proto. This is necessary as the conversion using the SeldonMessage proto
changes the Numeric types of all ints in a JSON into Floats.
Parameters
----------
user_model
Client user class
is_request
Whether this is part of the request flow as opposed to the response flow
client_request_raw
The request received in JSON format
client_raw_response
The raw client response from their model
Returns
-------
A SeldonMessage JSON response
"""
response = {}
if isinstance(client_raw_response, dict):
response["jsonData"] = client_raw_response
elif isinstance(client_raw_response, (bytes, bytearray)):
base64_data = base64.b64encode(client_raw_response)
response["binData"] = base64_data.decode("utf-8")
elif isinstance(client_raw_response, str):
response["strData"] = client_raw_response
else:
is_np = isinstance(client_raw_response, np.ndarray)
is_list = isinstance(client_raw_response, list)
if not (is_np or is_list):
raise SeldonMicroserviceException(
"Unknown data type returned as payload (must be list or np array):"
+ str(client_raw_response))
if is_np:
np_client_raw_response = client_raw_response
list_client_raw_response = client_raw_response.tolist()
else:
np_client_raw_response = np.array(client_raw_response)
list_client_raw_response = client_raw_response
response["data"] = {}
if "data" in client_request_raw:
if np.issubdtype(np_client_raw_response.dtype, np.number):
if "tensor" in client_request_raw["data"]:
default_data_type = "tensor"
result_client_response = {
"values": np_client_raw_response.ravel().tolist(),
"shape": np_client_raw_response.shape,
}
elif "tftensor" in client_request_raw["data"]:
default_data_type = "tftensor"
tf_json_str = json_format.MessageToJson(
tf.make_tensor_proto(np_client_raw_response))
result_client_response = json.loads(tf_json_str)
else:
default_data_type = "ndarray"
result_client_response = list_client_raw_response
else:
default_data_type = "ndarray"
result_client_response = list_client_raw_response
else:
if np.issubdtype(np_client_raw_response.dtype, np.number):
default_data_type = "tensor"
result_client_response = {
"values": np_client_raw_response.ravel().tolist(),
"shape": np_client_raw_response.shape,
}
else:
default_data_type = "ndarray"
result_client_response = list_client_raw_response
response["data"][default_data_type] = result_client_response
if is_request:
req_names = client_request_raw.get("data", {}).get("names", [])
names = client_feature_names(user_model, req_names)
else:
names = client_class_names(user_model, np_client_raw_response)
response["data"]["names"] = names
response["meta"] = {}
if meta:
tags = meta.get("tags", {})
else:
tags = {}
custom_tags = client_custom_tags(user_model)
if custom_tags:
tags.update(custom_tags)
if tags:
response["meta"]["tags"] = tags
metrics = client_custom_metrics(user_model)
if metrics:
response["meta"]["metrics"] = metrics
puid = client_request_raw.get("meta", {}).get("puid", None)
if puid:
response["meta"]["puid"] = puid
return response
def test_component_ok():
c = Component(True)
assert client_custom_metrics(c, SeldonMetrics()) == c.metrics()
def predict(
user_model: Any,
request: Union[prediction_pb2.SeldonMessage, List, Dict],
seldon_metrics: SeldonMetrics,
) -> Union[prediction_pb2.SeldonMessage, List, Dict]:
"""
Call the user model to get a prediction and package the response
Parameters
----------
user_model
User defined class instance
request
The incoming request
Returns
-------
The prediction
"""
is_proto = isinstance(request, prediction_pb2.SeldonMessage)
if hasattr(user_model, "predict_rest") and not is_proto:
logger.warning("predict_rest is deprecated. Please use predict_raw")
return user_model.predict_rest(request)
elif hasattr(user_model, "predict_grpc") and is_proto:
logger.warning("predict_grpc is deprecated. Please use predict_raw")
return user_model.predict_grpc(request)
else:
if hasattr(user_model, "predict_raw"):
try:
response = user_model.predict_raw(request)
if is_proto:
metrics = seldon_message_to_json(response.meta).get(
"metrics", [])
else:
metrics = response.get("meta", {}).get("metrics", [])
seldon_metrics.update(metrics)
return response
except SeldonNotImplementedError:
pass
if is_proto:
(features, meta, datadef,
data_type) = extract_request_parts(request)
client_response = client_predict(user_model,
features,
datadef.names,
meta=meta)
metrics = client_custom_metrics(user_model)
if seldon_metrics is not None:
seldon_metrics.update(metrics)
return construct_response(user_model, False, request,
client_response, meta, metrics)
else:
(features, meta, datadef,
data_type) = extract_request_parts_json(request)
class_names = datadef[
"names"] if datadef and "names" in datadef else []
client_response = client_predict(user_model,
features,
class_names,
meta=meta)
metrics = client_custom_metrics(user_model)
if seldon_metrics is not None:
seldon_metrics.update(metrics)
return construct_response_json(user_model, False, request,
client_response, meta, metrics)
def test_component_bad():
with pytest.raises(SeldonMicroserviceException):
c = Component(False)
client_custom_metrics(c)
