def index():
#try:
content = request.get_json(force=True)
requestPart = dict_digger.dig(content, 'request')
req_elements = None
if not requestPart is None:
requestCopy = requestPart.copy()
if "date" in requestCopy:
del requestCopy["date"]
requestMsg = json_to_seldon_message(requestCopy)
(req_features, _, req_datadef,
req_datatype) = extract_request_parts(requestMsg)
req_elements = createElelmentsArray(req_features,
list(req_datadef.names))
responsePart = dict_digger.dig(content, 'response')
res_elements = None
if not responsePart is None:
responseCopy = responsePart.copy()
if "date" in responseCopy:
del responseCopy["date"]
responseMsg = json_to_seldon_message(responseCopy)
(res_features, _, res_datadef,
res_datatype) = extract_request_parts(responseMsg)
res_elements = createElelmentsArray(res_features,
list(res_datadef.names))
if not req_elements is None and not res_elements is None:
for i, (a, b) in enumerate(zip(req_elements, res_elements)):
merged = {**a, **b}
content["elements"] = merged
reqJson = extractRow(i, requestMsg, req_datatype, req_features,
req_datadef)
resJson = extractRow(i, responseMsg, res_datatype, res_features,
res_datadef)
content["request"] = reqJson
content["response"] = resJson
#log formatted json to stdout for fluentd collection
print(str(json.dumps(content)))
elif not req_elements is None:
for i, e in enumerate(req_elements):
content["elements"] = e
reqJson = extractRow(i, requestMsg, req_datatype, req_features,
req_datadef)
content["request"] = reqJson
print(str(json.dumps(content)))
elif not res_elements is None:
for i, e in enumerate(res_elements):
content["elements"] = e
resJson = extractRow(i, responseMsg, res_datatype, res_features,
res_datadef)
content["response"] = resJson
print(str(json.dumps(content)))
else:
print(str(json.dumps(content)))
sys.stdout.flush()
return str(content)
def send_feedback(
user_model: Any,
request: prediction_pb2.Feedback,
predictive_unit_id: str,
seldon_metrics: SeldonMetrics,
) -> prediction_pb2.SeldonMessage:
"""
Parameters
----------
user_model
A Seldon user model
request
SeldonMesage proto
predictive_unit_id
The ID of the enclosing container predictive unit. Will be taken from environment.
Returns
-------
"""
seldon_metrics.update_reward(request.reward)
if hasattr(user_model, "send_feedback_rest"):
logger.warning(
"send_feedback_rest is deprecated. Please use send_feedback_raw")
request_json = json_format.MessageToJson(request)
response_json = user_model.send_feedback_rest(request_json)
return json_to_seldon_message(response_json)
elif hasattr(user_model, "send_feedback_grpc"):
logger.warning(
"send_feedback_grpc is deprecated. Please use send_feedback_raw")
response_json = user_model.send_feedback_grpc(request)
return json_to_seldon_message(response_json)
else:
if hasattr(user_model, "send_feedback_raw"):
try:
return user_model.send_feedback_raw(request)
except SeldonNotImplementedError:
pass
(datadef_request, features, truth,
reward) = extract_feedback_request_parts(request)
routing = request.response.meta.routing.get(predictive_unit_id)
client_response = client_send_feedback(user_model, features,
datadef_request.names, reward,
truth, routing)
if client_response is None:
client_response = np.array([])
else:
client_response = np.array(client_response)
return construct_response(user_model, False, request.request,
client_response)
def predict_raw(self, msg):
msg = json_to_seldon_message(msg)
if self.check_name == "img":
file_data = msg.binData
img = Image.open(io.BytesIO(file_data))
img.verify()
return {
"meta": seldon_message_to_json(msg.meta),
"data": {
"ndarray": [rs232_checksum(file_data).decode("utf-8")]
},
}
elif self.check_name == "txt":
file_data = msg.binData
return {
"meta": seldon_message_to_json(msg.meta),
"data": {
"ndarray": [file_data.decode("utf-8")]
},
}
elif self.check_name == "strData":
file_data = msg.strData
return {
"meta": seldon_message_to_json(msg.meta),
"data": {
"ndarray": [file_data]
},
}
def process_content(content):
if content is None:
return content
#no transformation using strData
if "strData" in content:
content["dataType"] = "text"
return content
#if we have dataType then have already parsed before
if "dataType" in content:
print('dataType already in content')
sys.stdout.flush()
return content
requestCopy = content.copy()
if "date" in requestCopy:
del requestCopy["date"]
requestMsg = json_to_seldon_message(requestCopy)
(req_features, _, req_datadef, req_datatype) = extract_request_parts(requestMsg)
elements = createElelmentsArray(req_features, list(req_datadef.names))
for i, e in enumerate(elements):
reqJson = extractRow(i, requestMsg, req_datatype, req_features, req_datadef)
reqJson["elements"] = e
content = reqJson
return content
def test_symmetric_json_conversion():
request_data = np.array([[5, 6, 7]])
datadef = scu.array_to_rest_datadef("ndarray", request_data)
json_request = {"jsonData": datadef}
seldon_message_request = scu.json_to_seldon_message(json_request)
result_json_request = scu.seldon_message_to_json(seldon_message_request)
assert json_request == result_json_request
def test_json_to_seldon_message_json_data():
data = {"jsonData": {"some": "value"}}
requestProto = scu.json_to_seldon_message(data)
assert len(requestProto.data.tensor.values) == 0
assert requestProto.WhichOneof("data_oneof") == "jsonData"
(json_data, meta, datadef, _) = scu.extract_request_parts(requestProto)
assert not isinstance(json_data, np.ndarray)
assert json_data == {"some": "value"}
def test_predict_grpc_proto_raw_data_seldon():
sc = SeldonClient(deployment_name="mymodel",
transport="grpc",
gateway="seldon")
proto_raw_data = json_to_seldon_message(RAW_DATA_TEST)
response = sc.predict(raw_data=proto_raw_data, client_return_type="proto")
request = seldon_message_to_json(response.request)
assert request == RAW_DATA_TEST
def test_json_to_seldon_message_str_data():
data = {"strData": "my string data"}
requestProto = scu.json_to_seldon_message(data)
assert len(requestProto.data.tensor.values) == 0
assert requestProto.WhichOneof("data_oneof") == "strData"
(arr, meta, datadef, _) = scu.extract_request_parts(requestProto)
assert not isinstance(arr, np.ndarray)
assert arr == "my string data"
def test_json_to_seldon_message_ndarray():
data = {"data": {"ndarray": [[1]]}}
requestProto = scu.json_to_seldon_message(data)
assert requestProto.data.ndarray[0][0] == 1
(arr, meta, datadef, _) = scu.extract_request_parts(requestProto)
assert isinstance(arr, np.ndarray)
assert arr.shape[0] == 1
assert arr.shape[1] == 1
assert arr[0][0] == 1
def Route():
requestJson = get_request()
logger.debug("REST Request: %s", request)
requestProto = json_to_seldon_message(requestJson)
logger.debug("Proto Request: %s", request)
responseProto = seldon_core.seldon_methods.route(
user_model, requestProto)
jsonDict = seldon_message_to_json(responseProto)
return jsonify(jsonDict)
def test_json_to_seldon_message_bin_data():
a = np.array([1, 2, 3])
serialized = pickle.dumps(a)
bdata_base64 = base64.b64encode(serialized).decode('utf-8')
data = {"binData": bdata_base64}
requestProto = scu.json_to_seldon_message(data)
assert len(requestProto.data.tensor.values) == 0
assert requestProto.WhichOneof("data_oneof") == "binData"
assert len(requestProto.binData) > 0
(arr, meta, datadef, _) = scu.extract_request_parts(requestProto)
assert not isinstance(arr, np.ndarray)
assert arr == serialized
def test_json_to_seldon_message_normal_data():
data = {"data": {"tensor": {"shape": [1, 1], "values": [1]}}}
requestProto = scu.json_to_seldon_message(data)
assert requestProto.data.tensor.values == [1]
assert requestProto.data.tensor.shape[0] == 1
assert requestProto.data.tensor.shape[1] == 1
assert len(requestProto.data.tensor.shape) == 2
(arr, meta, datadef, _) = scu.extract_request_parts(requestProto)
assert isinstance(arr, np.ndarray)
assert arr.shape[0] == 1
assert arr.shape[1] == 1
assert arr[0][0] == 1
def predict_raw(self, msg):
msg = json_to_seldon_message(msg)
if self.check_name == 'img':
file_data = msg.binData
img = Image.open(io.BytesIO(file_data))
img.verify()
return {
"meta": seldon_message_to_json(msg.meta),
"data": {
"ndarray": [rs232_checksum(file_data).decode('utf-8')]
},
"status": {
"code": 400,
"status": "FAILURE"
}
}
elif self.check_name == 'txt':
file_data = msg.binData
return {
"meta": seldon_message_to_json(msg.meta),
"data": {
"ndarray": [file_data.decode('utf-8')]
},
"status": {
"code": 400,
"status": "FAILURE"
}
}
elif self.check_name == 'strData':
file_data = msg.strData
return {
"meta": seldon_message_to_json(msg.meta),
"data": {
"ndarray": [file_data]
},
"status": {
"code": 400,
"status": "FAILURE"
}
}
def predict_raw(self, request):
log.debug("Predict raw")
request_data_type = request.WhichOneof("data_oneof")
default_data_type = request.data.WhichOneof("data_oneof")
log.debug(str(request_data_type), str(default_data_type))
if default_data_type == "tftensor" and self.grpc:
tfrequest = predict_pb2.PredictRequest()
tfrequest.model_spec.name = self.model_name
tfrequest.model_spec.signature_name = self.signature_name
tfrequest.inputs[self.model_input].CopyFrom(request.data.tftensor)
result = self.stub.Predict(tfrequest)
log.debug(result)
datadef = prediction_pb2.DefaultData(
tftensor=result.outputs[self.model_output])
return prediction_pb2.SeldonMessage(data=datadef)
elif request_data_type == "jsonData":
features = get_data_from_proto(request)
predictions = self.predict(features, features_names=[])
try:
sm = json_to_seldon_message({"jsonData": predictions})
except ParseError as e:
sm = prediction_pb2.SeldonMessage(strData=predictions)
return sm
else:
features = get_data_from_proto(request)
datadef = request.data
predictions = self.predict(features, datadef.names)
predictions = np.array(predictions)
if request_data_type is not "data":
default_data_type = "tensor"
class_names = []
data = array_to_grpc_datadef(predictions, class_names,
default_data_type)
return prediction_pb2.SeldonMessage(data=data)
def extract_data_part(content):
copy = content.copy()
# if 'instances' in body then tensorflow request protocol
# if 'predictions' then tensorflow response
# otherwise can use seldon logic for parsing and inferring type (won't be in here if outlier)
if "instances" in copy:
copy["instance"] = copy["instances"]
content_np = np.array(copy["instance"])
copy["dataType"] = "tabular"
first_element = content_np.item(0)
set_datatype_from_numpy(content_np, copy, first_element)
del copy["instances"]
elif "predictions" in copy:
copy["instance"] = copy["predictions"]
content_np = np.array(copy["predictions"])
copy["dataType"] = "tabular"
first_element = content_np.item(0)
set_datatype_from_numpy(content_np, copy, first_element)
del copy["predictions"]
else:
requestMsg = json_to_seldon_message(copy)
(req_features, _, req_datadef,
req_datatype) = extract_request_parts(requestMsg)
#set sensible defaults for non-tabular dataTypes
#tabular should be iterable and get inferred through later block
if req_datatype == "strData":
copy["dataType"] = "text"
if req_datatype == "binData":
copy["dataType"] = "image"
if isinstance(req_features, Iterable):
elements = createElelmentsArray(req_features,
list(req_datadef.names))
if isinstance(elements, Iterable):
for i, e in enumerate(elements):
reqJson = extractRow(i, requestMsg, req_datatype,
req_features, req_datadef)
reqJson["elements"] = e
copy = reqJson
copy["instance"] = json.loads(
json.dumps(req_features, cls=log_helper.NumpyEncoder))
if isinstance(req_features, np.ndarray):
set_datatype_from_numpy(req_features, copy, req_features.item(0))
#copy names into its own section of request
if "data" in content:
if "names" in content["data"]:
copy["names"] = content["data"]["names"]
#should now have processed content of seldon message so don't want its various constructs on top-level anymore
if "data" in copy:
del copy["data"]
if "strData" in copy:
del copy["strData"]
if "jsonData" in copy:
del copy["jsonData"]
if "binData" in copy:
del copy["binData"]
copy['payload'] = content
return copy
def mocked_requests_post_success_json_data(url, *args, **kwargs):
request = json_to_seldon_message({"jsonData": JSON_TEST_DATA})
json = seldon_message_to_json(request)
return MockResponse(json, 200, text="{}")
def test_json_to_seldon_message_bad_data():
with pytest.raises(SeldonMicroserviceException):
data = {"foo": "bar"}
requestProto = scu.json_to_seldon_message(data)
def extract_data_part(content, headers, message_type):
copy = content.copy()
namespace = log_helper.get_header(log_helper.NAMESPACE_HEADER_NAME,
headers)
inferenceservice_name = log_helper.get_header(
log_helper.INFERENCESERVICE_HEADER_NAME, headers)
endpoint_name = log_helper.get_header(log_helper.ENDPOINT_HEADER_NAME,
headers)
serving_engine = log_helper.serving_engine(headers)
# if 'instances' in body then tensorflow request protocol
# if 'predictions' then tensorflow response
# if 'model_name' and 'outputs' then v2 dataplane response
# if 'inputs' then v2 data plane request
# otherwise can use seldon logic for parsing and inferring type (won't be in here if outlier)
# V2 Data Plane Response
if "model_name" in copy and "outputs" in copy:
# assumes single output
output = copy["outputs"][0]
data_type = output["datatype"]
shape = output["shape"]
data = output["data"]
if data_type == "BYTES":
copy["dataType"] = "text"
copy["instance"] = array.array('B', data).tostring()
else:
arr = create_np_from_v2(data, data_type, shape)
copy["dataType"] = "tabular"
first_element = arr.item(0)
set_datatype_from_numpy(arr, copy, first_element)
copy["instance"] = arr.tolist()
del copy["outputs"]
del copy["model_name"]
del copy["model_version"]
elif "inputs" in copy:
# assumes single input
inputs = copy["inputs"][0]
data_type = inputs["datatype"]
shape = inputs["shape"]
data = inputs["data"]
if data_type == "BYTES":
copy["dataType"] = "text"
copy["instance"] = array.array('B', data).tostring()
else:
arr = create_np_from_v2(data, data_type, shape)
copy["dataType"] = "tabular"
first_element = arr.item(0)
set_datatype_from_numpy(arr, copy, first_element)
copy["instance"] = arr.tolist()
del copy["inputs"]
elif "instances" in copy:
copy["instance"] = copy["instances"]
content_np = np.array(copy["instance"])
copy["dataType"] = "tabular"
first_element = content_np.item(0)
set_datatype_from_numpy(content_np, copy, first_element)
elements = createElelmentsArray(content_np, None, namespace,
serving_engine, inferenceservice_name,
endpoint_name, message_type)
copy["elements"] = elements
del copy["instances"]
elif "predictions" in copy:
copy["instance"] = copy["predictions"]
content_np = np.array(copy["predictions"])
copy["dataType"] = "tabular"
first_element = content_np.item(0)
set_datatype_from_numpy(content_np, copy, first_element)
elements = createElelmentsArray(content_np, None, namespace,
serving_engine, inferenceservice_name,
endpoint_name, message_type)
copy["elements"] = elements
del copy["predictions"]
else:
requestMsg = json_to_seldon_message(copy)
(req_features, _, req_datadef,
req_datatype) = extract_request_parts(requestMsg)
# set sensible defaults for non-tabular dataTypes
# tabular should be iterable and get inferred through later block
if req_datatype == "strData":
copy["dataType"] = "text"
if req_datatype == "jsonData":
copy["dataType"] = "json"
if req_datatype == "binData":
copy["dataType"] = "image"
if isinstance(req_features, Iterable):
elements = createElelmentsArray(req_features,
list(req_datadef.names), namespace,
serving_engine,
inferenceservice_name,
endpoint_name, message_type)
copy["elements"] = elements
copy["instance"] = json.loads(
json.dumps(req_features, cls=log_helper.NumpyEncoder))
if isinstance(req_features, np.ndarray):
set_datatype_from_numpy(req_features, copy, req_features.item(0))
# copy names into its own section of request
if "data" in content:
if "names" in content["data"]:
copy["names"] = content["data"]["names"]
# should now have processed content of seldon message so don't want its various constructs on top-level anymore
if "data" in copy:
del copy["data"]
if "strData" in copy:
del copy["strData"]
if "jsonData" in copy:
del copy["jsonData"]
if "binData" in copy:
del copy["binData"]
copy["payload"] = content
return copy
