def index():
#try:
content = request.get_json(force=True)
requestPart = dict_digger.dig(content, 'request')
req_elements = None
if not requestPart is None:
requestCopy = requestPart.copy()
if "date" in requestCopy:
del requestCopy["date"]
requestMsg = json_to_seldon_message(requestCopy)
(req_features, _, req_datadef,
req_datatype) = extract_request_parts(requestMsg)
req_elements = createElelmentsArray(req_features,
list(req_datadef.names))
responsePart = dict_digger.dig(content, 'response')
res_elements = None
if not responsePart is None:
responseCopy = responsePart.copy()
if "date" in responseCopy:
del responseCopy["date"]
responseMsg = json_to_seldon_message(responseCopy)
(res_features, _, res_datadef,
res_datatype) = extract_request_parts(responseMsg)
res_elements = createElelmentsArray(res_features,
list(res_datadef.names))
if not req_elements is None and not res_elements is None:
for i, (a, b) in enumerate(zip(req_elements, res_elements)):
merged = {**a, **b}
content["elements"] = merged
reqJson = extractRow(i, requestMsg, req_datatype, req_features,
req_datadef)
resJson = extractRow(i, responseMsg, res_datatype, res_features,
res_datadef)
content["request"] = reqJson
content["response"] = resJson
#log formatted json to stdout for fluentd collection
print(str(json.dumps(content)))
elif not req_elements is None:
for i, e in enumerate(req_elements):
content["elements"] = e
reqJson = extractRow(i, requestMsg, req_datatype, req_features,
req_datadef)
content["request"] = reqJson
print(str(json.dumps(content)))
elif not res_elements is None:
for i, e in enumerate(res_elements):
content["elements"] = e
resJson = extractRow(i, responseMsg, res_datatype, res_features,
res_datadef)
content["response"] = resJson
print(str(json.dumps(content)))
else:
print(str(json.dumps(content)))
sys.stdout.flush()
return str(content)
def process_content(content):
if content is None:
return content
#no transformation using strData
if "strData" in content:
content["dataType"] = "text"
return content
#if we have dataType then have already parsed before
if "dataType" in content:
print('dataType already in content')
sys.stdout.flush()
return content
requestCopy = content.copy()
if "date" in requestCopy:
del requestCopy["date"]
requestMsg = json_to_seldon_message(requestCopy)
(req_features, _, req_datadef, req_datatype) = extract_request_parts(requestMsg)
elements = createElelmentsArray(req_features, list(req_datadef.names))
for i, e in enumerate(elements):
reqJson = extractRow(i, requestMsg, req_datatype, req_features, req_datadef)
reqJson["elements"] = e
content = reqJson
return content
def route(
user_model: Any, request: Union[prediction_pb2.SeldonMessage, List, Dict]
) -> Union[prediction_pb2.SeldonMessage, List, Dict]:
"""
Parameters
----------
user_model
A Seldon user model
request
A SelodonMessage proto
Returns
-------
"""
is_proto = isinstance(request, prediction_pb2.SeldonMessage)
if hasattr(user_model, "route_rest"):
logger.warning("route_rest is deprecated. Please use route_raw")
return user_model.route_rest(request)
elif hasattr(user_model, "route_grpc"):
logger.warning("route_grpc is deprecated. Please use route_raw")
return user_model.route_grpc(request)
else:
if hasattr(user_model, "route_raw"):
try:
return user_model.route_raw(request)
except SeldonNotImplementedError:
pass
if is_proto:
(features, meta, datadef,
data_type) = extract_request_parts(request)
client_response = client_route(user_model,
features,
datadef.names,
meta=meta)
if not isinstance(client_response, int):
raise SeldonMicroserviceException(
"Routing response must be int but got " +
str(client_response))
client_response_arr = np.array([[client_response]])
return construct_response(user_model, False, request,
client_response_arr)
else:
(features, meta, datadef,
data_type) = extract_request_parts_json(request)
class_names = datadef[
"names"] if datadef and "names" in datadef else []
client_response = client_route(user_model,
features,
class_names,
meta=meta)
if not isinstance(client_response, int):
raise SeldonMicroserviceException(
"Routing response must be int but got " +
str(client_response))
client_response_arr = np.array([[client_response]])
return construct_response_json(user_model, False, request,
client_response_arr)
def test_json_to_seldon_message_str_data():
data = {"strData": "my string data"}
requestProto = scu.json_to_seldon_message(data)
assert len(requestProto.data.tensor.values) == 0
assert requestProto.WhichOneof("data_oneof") == "strData"
(arr, meta, datadef, _) = scu.extract_request_parts(requestProto)
assert not isinstance(arr, np.ndarray)
assert arr == "my string data"
def test_json_to_seldon_message_json_data():
data = {"jsonData": {"some": "value"}}
requestProto = scu.json_to_seldon_message(data)
assert len(requestProto.data.tensor.values) == 0
assert requestProto.WhichOneof("data_oneof") == "jsonData"
(json_data, meta, datadef, _) = scu.extract_request_parts(requestProto)
assert not isinstance(json_data, np.ndarray)
assert json_data == {"some": "value"}
def transform_output(
user_model: Any,
request: Union[prediction_pb2.SeldonMessage, List, Dict]) \
-> Union[prediction_pb2.SeldonMessage, List, Dict]:
"""
Parameters
----------
user_model
User defined class to handle transform input
request
The incoming request
Returns
-------
The transformed request
"""
is_proto = isinstance(request, prediction_pb2.SeldonMessage)
if hasattr(user_model, "transform_output_rest"):
logger.warning(
"transform_input_rest is deprecated. Please use transform_input_raw"
)
return user_model.transform_output_rest(request)
elif hasattr(user_model, "transform_output_grpc"):
logger.warning(
"transform_input_grpc is deprecated. Please use transform_input_raw"
)
return user_model.transform_output_grpc(request)
else:
if hasattr(user_model, "transform_output_raw"):
try:
return user_model.transform_output_raw(request)
except SeldonNotImplementedError:
pass
if is_proto:
(features, meta, datadef,
data_type) = extract_request_parts(request)
client_response = client_transform_output(user_model,
features,
datadef.names,
meta=meta)
return construct_response(user_model, False, request,
client_response)
else:
(features, meta, datadef,
data_type) = extract_request_parts_json(request)
class_names = datadef[
"names"] if datadef and "names" in datadef else []
client_response = client_transform_output(user_model,
features,
class_names,
meta=meta)
return construct_response_json(user_model, False, request,
client_response)
def test_json_to_seldon_message_ndarray():
data = {"data": {"ndarray": [[1]]}}
requestProto = scu.json_to_seldon_message(data)
assert requestProto.data.ndarray[0][0] == 1
(arr, meta, datadef, _) = scu.extract_request_parts(requestProto)
assert isinstance(arr, np.ndarray)
assert arr.shape[0] == 1
assert arr.shape[1] == 1
assert arr[0][0] == 1
def test_json_to_seldon_message_bin_data():
a = np.array([1, 2, 3])
serialized = pickle.dumps(a)
bdata_base64 = base64.b64encode(serialized).decode('utf-8')
data = {"binData": bdata_base64}
requestProto = scu.json_to_seldon_message(data)
assert len(requestProto.data.tensor.values) == 0
assert requestProto.WhichOneof("data_oneof") == "binData"
assert len(requestProto.binData) > 0
(arr, meta, datadef, _) = scu.extract_request_parts(requestProto)
assert not isinstance(arr, np.ndarray)
assert arr == serialized
def test_json_to_seldon_message_normal_data():
data = {"data": {"tensor": {"shape": [1, 1], "values": [1]}}}
requestProto = scu.json_to_seldon_message(data)
assert requestProto.data.tensor.values == [1]
assert requestProto.data.tensor.shape[0] == 1
assert requestProto.data.tensor.shape[1] == 1
assert len(requestProto.data.tensor.shape) == 2
(arr, meta, datadef, _) = scu.extract_request_parts(requestProto)
assert isinstance(arr, np.ndarray)
assert arr.shape[0] == 1
assert arr.shape[1] == 1
assert arr[0][0] == 1
def predict(
user_model: Any, request: Union[prediction_pb2.SeldonMessage, List, Dict]
) -> Union[prediction_pb2.SeldonMessage, List, Dict]:
"""
Call the user model to get a prediction and package the response
Parameters
----------
user_model
User defined class instance
request
The incoming request
Returns
-------
The prediction
"""
is_proto = isinstance(request, prediction_pb2.SeldonMessage)
if hasattr(user_model, "predict_rest") and not is_proto:
logger.warning("predict_rest is deprecated. Please use predict_raw")
return user_model.predict_rest(request)
elif hasattr(user_model, "predict_grpc") and is_proto:
logger.warning("predict_grpc is deprecated. Please use predict_raw")
return user_model.predict_grpc(request)
else:
if hasattr(user_model, "predict_raw"):
try:
return user_model.predict_raw(request)
except SeldonNotImplementedError:
pass
if is_proto:
(features, meta, datadef,
data_type) = extract_request_parts(request)
client_response = client_predict(user_model,
features,
datadef.names,
meta=meta)
return construct_response(user_model, False, request,
client_response)
else:
(features, meta, datadef,
data_type) = extract_request_parts_json(request)
class_names = datadef[
"names"] if datadef and "names" in datadef else []
client_response = client_predict(user_model,
features,
class_names,
meta=meta)
return construct_response_json(user_model, False, request,
client_response)
def test_create_grpc_reponse_text_ndarray():
user_model = UserObject()
request_data = np.array([["hello", "world"], ["hello", "another", "world"]])
datadef = scu.array_to_grpc_datadef("ndarray", request_data)
request = prediction_pb2.SeldonMessage(data=datadef)
(features, meta, datadef, data_type) = scu.extract_request_parts(request)
raw_response = np.array([["hello", "world"], ["here", "another"]])
sm = scu.construct_response(user_model, True, request, raw_response)
assert sm.data.WhichOneof("data_oneof") == "ndarray"
assert type(features[0]) == list
assert np.array_equal(sm.data.ndarray, raw_response)
assert datadef == request.data
assert np.array_equal(features, request_data)
assert data_type == "data"
def extract_data_part(content):
copy = content.copy()
# if 'instances' in body then tensorflow request protocol
# if 'predictions' then tensorflow response
# otherwise can use seldon logic for parsing and inferring type (won't be in here if outlier)
if "instances" in copy:
copy["instance"] = copy["instances"]
content_np = np.array(copy["instance"])
copy["dataType"] = "tabular"
first_element = content_np.item(0)
set_datatype_from_numpy(content_np, copy, first_element)
del copy["instances"]
elif "predictions" in copy:
copy["instance"] = copy["predictions"]
content_np = np.array(copy["predictions"])
copy["dataType"] = "tabular"
first_element = content_np.item(0)
set_datatype_from_numpy(content_np, copy, first_element)
del copy["predictions"]
else:
requestMsg = json_to_seldon_message(copy)
(req_features, _, req_datadef,
req_datatype) = extract_request_parts(requestMsg)
#set sensible defaults for non-tabular dataTypes
#tabular should be iterable and get inferred through later block
if req_datatype == "strData":
copy["dataType"] = "text"
if req_datatype == "binData":
copy["dataType"] = "image"
if isinstance(req_features, Iterable):
elements = createElelmentsArray(req_features,
list(req_datadef.names))
if isinstance(elements, Iterable):
for i, e in enumerate(elements):
reqJson = extractRow(i, requestMsg, req_datatype,
req_features, req_datadef)
reqJson["elements"] = e
copy = reqJson
copy["instance"] = json.loads(
json.dumps(req_features, cls=log_helper.NumpyEncoder))
if isinstance(req_features, np.ndarray):
set_datatype_from_numpy(req_features, copy, req_features.item(0))
#copy names into its own section of request
if "data" in content:
if "names" in content["data"]:
copy["names"] = content["data"]["names"]
#should now have processed content of seldon message so don't want its various constructs on top-level anymore
if "data" in copy:
del copy["data"]
if "strData" in copy:
del copy["strData"]
if "jsonData" in copy:
del copy["jsonData"]
if "binData" in copy:
del copy["binData"]
copy['payload'] = content
return copy
def aggregate(
user_model: Any, request: Union[prediction_pb2.SeldonMessageList, List,
Dict]
) -> Union[prediction_pb2.SeldonMessage, List, Dict]:
"""
Aggregate a list of payloads
Parameters
----------
user_model
A Seldon user model
request
SeldonMessage proto
Returns
-------
Aggregated SeldonMessage proto
"""
is_proto = isinstance(request, prediction_pb2.SeldonMessageList)
if hasattr(user_model, "aggregate_rest"):
logger.warning(
"aggregate_rest is deprecated. Please use aggregate_raw")
return user_model.aggregate_rest(request)
elif hasattr(user_model, "aggregate_grpc"):
logger.warning(
"aggregate_grpc is deprecated. Please use aggregate_raw")
return user_model.aggregate_grpc(request)
else:
if hasattr(user_model, "aggregate_raw"):
try:
return user_model.aggregate_raw(request)
except SeldonNotImplementedError:
pass
if is_proto:
features_list = []
names_list = []
for msg in request.seldonMessages:
(features, meta, datadef,
data_type) = extract_request_parts(msg)
features_list.append(features)
names_list.append(datadef.names)
client_response = client_aggregate(user_model, features_list,
names_list)
return construct_response(user_model, False,
request.seldonMessages[0],
client_response)
else:
features_list = []
names_list = []
if isinstance(request, list):
msgs = request
elif "seldonMessages" in request and isinstance(
request["seldonMessages"], list):
msgs = request["seldonMessages"]
else:
raise SeldonMicroserviceException(
f"Invalid request data type: {request}")
for msg in msgs:
(features, meta, datadef,
data_type) = extract_request_parts_json(msg)
class_names = datadef[
"names"] if datadef and "names" in datadef else []
features_list.append(features)
names_list.append(class_names)
client_response = client_aggregate(user_model, features_list,
names_list)
return construct_response_json(user_model, False, msgs[0],
client_response)
def transform_input(
user_model: Any,
request: Union[prediction_pb2.SeldonMessage, List, Dict],
seldon_metrics: SeldonMetrics,
) -> Union[prediction_pb2.SeldonMessage, List, Dict]:
"""
Parameters
----------
user_model
User defined class to handle transform input
request
The incoming request
Returns
-------
The transformed request
"""
is_proto = isinstance(request, prediction_pb2.SeldonMessage)
if hasattr(user_model, "transform_input_rest"):
logger.warning(
"transform_input_rest is deprecated. Please use transform_input_raw"
)
return user_model.transform_input_rest(request)
elif hasattr(user_model, "transform_input_grpc"):
logger.warning(
"transform_input_grpc is deprecated. Please use transform_input_raw"
)
return user_model.transform_input_grpc(request)
else:
if hasattr(user_model, "transform_input_raw"):
try:
response = user_model.transform_input_raw(request)
if is_proto:
metrics = seldon_message_to_json(response.meta).get(
"metrics", [])
else:
metrics = response.get("meta", {}).get("metrics", [])
seldon_metrics.update(metrics)
return response
except SeldonNotImplementedError:
pass
if is_proto:
(features, meta, datadef,
data_type) = extract_request_parts(request)
client_response = client_transform_input(user_model,
features,
datadef.names,
meta=meta)
metrics = client_custom_metrics(user_model)
if seldon_metrics is not None:
seldon_metrics.update(metrics)
return construct_response(user_model, False, request,
client_response, meta, metrics)
else:
(features, meta, datadef,
data_type) = extract_request_parts_json(request)
class_names = datadef[
"names"] if datadef and "names" in datadef else []
client_response = client_transform_input(user_model,
features,
class_names,
meta=meta)
metrics = client_custom_metrics(user_model)
if seldon_metrics is not None:
seldon_metrics.update(metrics)
return construct_response_json(user_model, False, request,
client_response, meta, metrics)
def extract_data_part(content, headers, message_type):
copy = content.copy()
namespace = log_helper.get_header(log_helper.NAMESPACE_HEADER_NAME,
headers)
inferenceservice_name = log_helper.get_header(
log_helper.INFERENCESERVICE_HEADER_NAME, headers)
endpoint_name = log_helper.get_header(log_helper.ENDPOINT_HEADER_NAME,
headers)
serving_engine = log_helper.serving_engine(headers)
# if 'instances' in body then tensorflow request protocol
# if 'predictions' then tensorflow response
# if 'model_name' and 'outputs' then v2 dataplane response
# if 'inputs' then v2 data plane request
# otherwise can use seldon logic for parsing and inferring type (won't be in here if outlier)
# V2 Data Plane Response
if "model_name" in copy and "outputs" in copy:
# assumes single output
output = copy["outputs"][0]
data_type = output["datatype"]
shape = output["shape"]
data = output["data"]
if data_type == "BYTES":
copy["dataType"] = "text"
copy["instance"] = array.array('B', data).tostring()
else:
arr = create_np_from_v2(data, data_type, shape)
copy["dataType"] = "tabular"
first_element = arr.item(0)
set_datatype_from_numpy(arr, copy, first_element)
copy["instance"] = arr.tolist()
del copy["outputs"]
del copy["model_name"]
del copy["model_version"]
elif "inputs" in copy:
# assumes single input
inputs = copy["inputs"][0]
data_type = inputs["datatype"]
shape = inputs["shape"]
data = inputs["data"]
if data_type == "BYTES":
copy["dataType"] = "text"
copy["instance"] = array.array('B', data).tostring()
else:
arr = create_np_from_v2(data, data_type, shape)
copy["dataType"] = "tabular"
first_element = arr.item(0)
set_datatype_from_numpy(arr, copy, first_element)
copy["instance"] = arr.tolist()
del copy["inputs"]
elif "instances" in copy:
copy["instance"] = copy["instances"]
content_np = np.array(copy["instance"])
copy["dataType"] = "tabular"
first_element = content_np.item(0)
set_datatype_from_numpy(content_np, copy, first_element)
elements = createElelmentsArray(content_np, None, namespace,
serving_engine, inferenceservice_name,
endpoint_name, message_type)
copy["elements"] = elements
del copy["instances"]
elif "predictions" in copy:
copy["instance"] = copy["predictions"]
content_np = np.array(copy["predictions"])
copy["dataType"] = "tabular"
first_element = content_np.item(0)
set_datatype_from_numpy(content_np, copy, first_element)
elements = createElelmentsArray(content_np, None, namespace,
serving_engine, inferenceservice_name,
endpoint_name, message_type)
copy["elements"] = elements
del copy["predictions"]
else:
requestMsg = json_to_seldon_message(copy)
(req_features, _, req_datadef,
req_datatype) = extract_request_parts(requestMsg)
# set sensible defaults for non-tabular dataTypes
# tabular should be iterable and get inferred through later block
if req_datatype == "strData":
copy["dataType"] = "text"
if req_datatype == "jsonData":
copy["dataType"] = "json"
if req_datatype == "binData":
copy["dataType"] = "image"
if isinstance(req_features, Iterable):
elements = createElelmentsArray(req_features,
list(req_datadef.names), namespace,
serving_engine,
inferenceservice_name,
endpoint_name, message_type)
copy["elements"] = elements
copy["instance"] = json.loads(
json.dumps(req_features, cls=log_helper.NumpyEncoder))
if isinstance(req_features, np.ndarray):
set_datatype_from_numpy(req_features, copy, req_features.item(0))
# copy names into its own section of request
if "data" in content:
if "names" in content["data"]:
copy["names"] = content["data"]["names"]
# should now have processed content of seldon message so don't want its various constructs on top-level anymore
if "data" in copy:
del copy["data"]
if "strData" in copy:
del copy["strData"]
if "jsonData" in copy:
del copy["jsonData"]
if "binData" in copy:
del copy["binData"]
copy["payload"] = content
return copy
def aggregate(
user_model: Any,
request: Union[prediction_pb2.SeldonMessageList, List, Dict],
seldon_metrics: SeldonMetrics,
) -> Union[prediction_pb2.SeldonMessage, List, Dict]:
"""
Aggregate a list of payloads
Parameters
----------
user_model
A Seldon user model
request
SeldonMessage proto
seldon_metrics
A SeldonMetrics instance
Returns
-------
Aggregated SeldonMessage proto
"""
def merge_meta(meta_list):
tags = {}
for meta in meta_list:
if meta:
tags.update(meta.get("tags", {}))
return {"tags": tags}
def merge_metrics(meta_list, custom_metrics):
metrics = []
for meta in meta_list:
if meta:
metrics.extend(meta.get("metrics", []))
metrics.extend(custom_metrics)
return metrics
is_proto = isinstance(request, prediction_pb2.SeldonMessageList)
if hasattr(user_model, "aggregate_rest"):
logger.warning(
"aggregate_rest is deprecated. Please use aggregate_raw")
return user_model.aggregate_rest(request)
elif hasattr(user_model, "aggregate_grpc"):
logger.warning(
"aggregate_grpc is deprecated. Please use aggregate_raw")
return user_model.aggregate_grpc(request)
else:
if hasattr(user_model, "aggregate_raw"):
try:
response = user_model.aggregate_raw(request)
handle_raw_custom_metrics(response, seldon_metrics, is_proto,
AGGREGATE_METRIC_METHOD_TAG)
return response
except SeldonNotImplementedError:
pass
if is_proto:
features_list = []
names_list = []
meta_list = []
for msg in request.seldonMessages:
(features, meta, datadef,
data_type) = extract_request_parts(msg)
features_list.append(features)
names_list.append(datadef.names)
meta_list.append(meta)
client_response = client_aggregate(user_model, features_list,
names_list)
metrics = client_custom_metrics(
user_model,
seldon_metrics,
AGGREGATE_METRIC_METHOD_TAG,
client_response.metrics,
)
return construct_response(
user_model,
False,
request.seldonMessages[0],
client_response.data,
merge_meta(meta_list),
merge_metrics(meta_list, metrics),
client_response.tags,
)
else:
features_list = []
names_list = []
if isinstance(request, list):
msgs = request
elif "seldonMessages" in request and isinstance(
request["seldonMessages"], list):
msgs = request["seldonMessages"]
else:
raise SeldonMicroserviceException(
f"Invalid request data type: {request}")
meta_list = []
for msg in msgs:
(features, meta, datadef,
data_type) = extract_request_parts_json(msg)
class_names = datadef[
"names"] if datadef and "names" in datadef else []
features_list.append(features)
names_list.append(class_names)
meta_list.append(meta)
client_response = client_aggregate(user_model, features_list,
names_list)
metrics = client_custom_metrics(
user_model,
seldon_metrics,
AGGREGATE_METRIC_METHOD_TAG,
client_response.metrics,
)
return construct_response_json(
user_model,
False,
msgs[0],
client_response.data,
merge_meta(meta_list),
merge_metrics(meta_list, metrics),
client_response.tags,
)
def predict(
user_model: Any,
request: Union[prediction_pb2.SeldonMessage, List, Dict, bytes],
seldon_metrics: SeldonMetrics,
) -> Union[prediction_pb2.SeldonMessage, List, Dict, bytes]:
"""
Call the user model to get a prediction and package the response
Parameters
----------
user_model
User defined class instance
request
The incoming request
Returns
-------
The prediction
"""
# TODO: Find a way to choose predict_rest or predict_grpc when payload is
# not decoded
is_proto = isinstance(request, prediction_pb2.SeldonMessage)
if hasattr(user_model, "predict_rest") and not is_proto:
logger.warning("predict_rest is deprecated. Please use predict_raw")
return user_model.predict_rest(request)
elif hasattr(user_model, "predict_grpc") and is_proto:
logger.warning("predict_grpc is deprecated. Please use predict_raw")
return user_model.predict_grpc(request)
else:
if hasattr(user_model, "predict_raw"):
try:
response = user_model.predict_raw(request)
handle_raw_custom_metrics(response, seldon_metrics, is_proto,
PREDICT_METRIC_METHOD_TAG)
return response
except SeldonNotImplementedError:
pass
if is_proto:
(features, meta, datadef,
data_type) = extract_request_parts(request)
client_response = client_predict(user_model,
features,
datadef.names,
meta=meta)
metrics = client_custom_metrics(
user_model,
seldon_metrics,
PREDICT_METRIC_METHOD_TAG,
client_response.metrics,
)
return construct_response(
user_model,
False,
request,
client_response.data,
meta,
metrics,
client_response.tags,
)
else:
(features, meta, datadef,
data_type) = extract_request_parts_json(request)
class_names = datadef[
"names"] if datadef and "names" in datadef else []
client_response = client_predict(user_model,
features,
class_names,
meta=meta)
metrics = client_custom_metrics(
user_model,
seldon_metrics,
PREDICT_METRIC_METHOD_TAG,
client_response.metrics,
)
return construct_response_json(
user_model,
False,
request,
client_response.data,
meta,
metrics,
client_response.tags,
)