def predict_raw(self, msg):
msg = json_to_seldon_message(msg)
if self.check_name == "img":
file_data = msg.binData
img = Image.open(io.BytesIO(file_data))
img.verify()
return {
"meta": seldon_message_to_json(msg.meta),
"data": {
"ndarray": [rs232_checksum(file_data).decode("utf-8")]
},
}
elif self.check_name == "txt":
file_data = msg.binData
return {
"meta": seldon_message_to_json(msg.meta),
"data": {
"ndarray": [file_data.decode("utf-8")]
},
}
elif self.check_name == "strData":
file_data = msg.strData
return {
"meta": seldon_message_to_json(msg.meta),
"data": {
"ndarray": [file_data]
},
}
def test_predict_grpc_raw_data_gateway():
sc = SeldonClient(deployment_name="mymodel",
transport="grpc",
gateway="istio")
response = sc.predict(raw_data=RAW_DATA_TEST, client_return_type="proto")
request = seldon_message_to_json(response.request)
assert request == RAW_DATA_TEST
def test_symmetric_json_conversion():
request_data = np.array([[5, 6, 7]])
datadef = scu.array_to_rest_datadef("ndarray", request_data)
json_request = {"jsonData": datadef}
seldon_message_request = scu.json_to_seldon_message(json_request)
result_json_request = scu.seldon_message_to_json(seldon_message_request)
assert json_request == result_json_request
def extractRow(i: int, requestMsg: prediction_pb2.SeldonMessage,
req_datatype: str, req_features: np.ndarray,
req_datadef: "prediction_pb2.SeldonMessage.data"):
datatyReq = "ndarray"
dataType = "tabular"
if len(req_features.shape) == 2:
dataReq = array_to_grpc_datadef(
datatyReq, np.expand_dims(req_features[i], axis=0),
req_datadef.names)
else:
if len(req_features.shape) > 2:
dataType = "image"
else:
dataType = "text"
req_features = np.char.decode(req_features.astype('S'), "utf-8")
dataReq = array_to_grpc_datadef(
datatyReq, np.expand_dims(req_features[i], axis=0),
req_datadef.names)
requestMsg2 = prediction_pb2.SeldonMessage(data=dataReq,
meta=requestMsg.meta)
reqJson = {}
reqJson["payload"] = seldon_message_to_json(requestMsg2)
# setting dataType here temporarily so calling method will be able to access it
# don't want to set it at the payload level
reqJson["dataType"] = dataType
return reqJson
def test_seldon_message_to_json():
arr = np.array([1, 2])
datadef = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr))
request = prediction_pb2.SeldonMessage(data=datadef)
dict = scu.seldon_message_to_json(request)
assert dict["data"]["tensor"]["values"] == [1, 2]
def make_seldon_prediction(serialized, model):
endpoint = ENDPOINT
kwargs = {
# SELDON API GATEWAY
# "deployment_name": model,
# "namespace": NAMESPACE,
# "namespace": model_params.get('namespace', NAMESPACE),
# "gateway_endpoint": endpoint,
# NAIVE API GATEWAY
"method": "predict",
"microservice_endpoint": endpoint,
"payload_type": "tensor",
# "bin_data": serialized,
"bin_data": serialized,
# grpc_max_send_message_length: int = 4 * 1024 * 1024,
# grpc_max_receive_message_length: int = 4 * 1024 * 1024,
# names: Iterable[str] = None,
}
# return rest_predict_gateway(**kwargs)
# return grpc_predict_gateway(**kwargs)
response = grpc_predict(**kwargs)
if response.success:
log.info('GRPC - Prediction request successful')
response = seldon_message_to_json(response.response)
shape, values = response['data'].get('tensor').values()
return np.array(values).reshape(shape[:-1])
else:
# log.info('REST request not succesfull.')
log.info('GRPC - Prediction request not succesfull')
def SendFeedback():
requestJson = get_request()
logger.debug("REST Request: %s", request)
requestProto = json_to_feedback(requestJson)
logger.debug("Proto Request: %s", requestProto)
responseProto = seldon_core.seldon_methods.send_feedback(
user_model, requestProto, PRED_UNIT_ID)
jsonDict = seldon_message_to_json(responseProto)
return jsonify(jsonDict)
def Aggregate():
requestJson = get_request()
logger.debug("REST Request: %s", request)
requestProto = json_to_seldon_messages(requestJson)
logger.debug("Proto Request: %s", request)
responseProto = seldon_core.seldon_methods.aggregate(
user_model, requestProto)
jsonDict = seldon_message_to_json(responseProto)
return jsonify(jsonDict)
def TransformOutput():
requestJson = get_request()
logger.debug("REST Request: %s", request)
requestProto = json_to_seldon_message(requestJson)
logger.debug("Proto Request: %s", request)
responseProto = seldon_core.seldon_methods.transform_output(
user_model, requestProto)
jsonDict = seldon_message_to_json(responseProto)
return jsonify(jsonDict)
def test_predict_rest_json_data_seldon(mock_post, mock_token):
sc = SeldonClient(deployment_name="mymodel", gateway="seldon")
response = sc.predict(json_data=JSON_TEST_DATA)
json_response = seldon_message_to_json(response.response)
assert "jsonData" in mock_post.call_args[1]["json"]
assert mock_post.call_args[1]["json"]["jsonData"] == JSON_TEST_DATA
assert response.success is True
assert json_response["jsonData"] == JSON_TEST_DATA
assert mock_post.call_count == 1
def test_predict_rest_json_data_ambassador(mock_post):
sc = SeldonClient(deployment_name="mymodel", gateway="ambassador")
response = sc.predict(json_data=JSON_TEST_DATA, client_return_type="proto")
json_response = seldon_message_to_json(response.response)
assert "jsonData" in mock_post.call_args[1]["json"]
assert mock_post.call_args[1]["json"]["jsonData"] == JSON_TEST_DATA
assert response.success is True
assert json_response["jsonData"] == JSON_TEST_DATA
assert mock_post.call_count == 1
def test_predict_rest_raw_data_seldon_proto(mock_post):
sc = SeldonClient(deployment_name="mymodel",
gateway="seldon",
client_return_type="proto")
response = sc.predict(raw_data=RAW_DATA_TEST)
json_response = seldon_message_to_json(response.response)
assert mock_post.call_args[1]["json"] == RAW_DATA_TEST
assert response.success is True
assert mock_post.call_count == 1
def test_predict_microservice_rest_json_data(mock_post):
sc = SeldonClient(deployment_name="mymodel")
response = sc.microservice(method="predict", json_data=JSON_TEST_DATA)
json_response = seldon_message_to_json(response.response)
assert "jsonData" in mock_post.call_args[1]["data"]["json"]
assert response.success is True
assert mock_post.call_args[1]["data"]["json"] == json.dumps(
{"jsonData": JSON_TEST_DATA})
assert json_response["jsonData"] == JSON_TEST_DATA
assert mock_post.call_count == 1
def test_explain_rest_json_data_ambassador(mock_post):
sc = SeldonClient(deployment_name="mymodel", gateway="ambassador")
response = sc.explain(json_data=JSON_TEST_DATA)
json_response = seldon_message_to_json(response.response)
# Currently this doesn't need to convert to JSON due to #1083
# i.e. json_response = seldon_message_to_json(response.response)
assert "jsonData" in mock_post.call_args[1]["json"]
assert mock_post.call_args[1]["json"]["jsonData"] == JSON_TEST_DATA
assert json_response["jsonData"] == JSON_TEST_DATA
assert mock_post.call_count == 1
def predict_raw(self, msg):
msg = json_to_seldon_message(msg)
if self.check_name == 'img':
file_data = msg.binData
img = Image.open(io.BytesIO(file_data))
img.verify()
return {
"meta": seldon_message_to_json(msg.meta),
"data": {
"ndarray": [rs232_checksum(file_data).decode('utf-8')]
},
"status": {
"code": 400,
"status": "FAILURE"
}
}
elif self.check_name == 'txt':
file_data = msg.binData
return {
"meta": seldon_message_to_json(msg.meta),
"data": {
"ndarray": [file_data.decode('utf-8')]
},
"status": {
"code": 400,
"status": "FAILURE"
}
}
elif self.check_name == 'strData':
file_data = msg.strData
return {
"meta": seldon_message_to_json(msg.meta),
"data": {
"ndarray": [file_data]
},
"status": {
"code": 400,
"status": "FAILURE"
}
}
def transform_output_raw(
self, request: Union[prediction_pb2.SeldonMessage, List, Dict]
) -> Union[prediction_pb2.SeldonMessage, List, Dict]:
is_proto = isinstance(request, prediction_pb2.SeldonMessage)
arr = np.array([9, 9])
datadef = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr))
response = prediction_pb2.SeldonMessage(data=datadef)
if is_proto:
return response
else:
return seldon_message_to_json(response)
def extractRow(i:int,requestMsg: prediction_pb2.SeldonMessage,req_datatype: str,req_features: np.ndarray,req_datadef: "prediction_pb2.SeldonMessage.data"):
datatyReq = "ndarray"
dataType = "tabular"
if len(req_features.shape) == 2:
dataReq = array_to_grpc_datadef(datatyReq, np.expand_dims(req_features[i], axis=0), req_datadef.names)
else:
if len(req_features.shape) > 2:
dataType="image"
else:
dataType="text"
req_features= np.char.decode(req_features.astype('S'),"utf-8")
dataReq = array_to_grpc_datadef(datatyReq, req_features, req_datadef.names)
requestMsg2 = prediction_pb2.SeldonMessage(data=dataReq, meta=requestMsg.meta)
reqJson = seldon_message_to_json(requestMsg2)
reqJson["dataType"] = dataType
return reqJson
def test_sklearn_server(data):
y_true = [5, 0]
sc = SeldonClient(
gateway="ambassador",
gateway_endpoint=API_AMBASSADOR,
deployment_name="image-classifier",
payload_type="ndarray",
namespace="seldon",
transport="rest",
)
sm_result = sc.predict(data=np.array(data))
logging.info(sm_result)
result = seldon_message_to_json(sm_result.response)
values = result.get("data", {}).get("ndarray", {})
assert values == labels
def extractRow(i: int, requestMsg: prediction_pb2.SeldonMessage,
req_datatype: str, req_features: np.ndarray,
req_datadef: prediction_pb2.SeldonMessage.data):
if req_datatype == "data":
datatyReq = requestMsg.data.WhichOneof("data_oneof")
else:
datatyReq = "ndarray"
if len(req_features.shape) == 2:
dataReq = array_to_grpc_datadef(
datatyReq, np.expand_dims(req_features[i], axis=0),
req_datadef.names)
else:
dataReq = array_to_grpc_datadef(datatyReq, req_features,
req_datadef.names)
requestMsg2 = prediction_pb2.SeldonMessage(data=dataReq,
meta=requestMsg.meta)
reqJson = seldon_message_to_json(requestMsg2)
return reqJson
def test_sklearn_server():
data = ["From: [email protected] (Brian Kantor)\nSubject: Re: HELP for Kidney Stones ..............\nOrganization: The Avant-Garde of the Now, Ltd.\nLines: 12\nNNTP-Posting-Host: ucsd.edu\n\nAs I recall from my bout with kidney stones, there isn't any\nmedication that can do anything about them except relieve the pain.\n\nEither they pass, or they have to be broken up with sound, or they have\nto be extracted surgically.\n\nWhen I was in, the X-ray tech happened to mention that she'd had kidney\nstones and children, and the childbirth hurt less.\n\nDemerol worked, although I nearly got arrested on my way home when I barfed\nall over the police car parked just outside the ER.\n\t- Brian\n",
'From: [email protected] (David Rind)\nSubject: Re: Candida(yeast) Bloom, Fact or Fiction\nOrganization: Beth Israel Hospital, Harvard Medical School, Boston Mass., USA\nLines: 37\nNNTP-Posting-Host: enterprise.bih.harvard.edu\n\nIn article <*****@*****.**>\n [email protected] writes:\n>are in a different class. The big question seems to be is it reasonable to \n>use them in patients with GI distress or sinus problems that *could* be due \n>to candida blooms following the use of broad-spectrum antibiotics?\n\nI guess I\'m still not clear on what the term "candida bloom" means,\nbut certainly it is well known that thrush (superficial candidal\ninfections on mucous membranes) can occur after antibiotic use.\nThis has nothing to do with systemic yeast syndrome, the "quack"\ndiagnosis that has been being discussed.\n\n\n>found in the sinus mucus membranes than is candida. Women have been known \n>for a very long time to suffer from candida blooms in the v****a and a \n>women is lucky to find a physician who is willing to treat the cause and \n>not give give her advise to use the OTC anti-fungal creams.\n\nLucky how? Since a recent article (randomized controlled trial) of\noral yogurt on reducing vaginal candidiasis, I\'ve mentioned to a \nnumber of patients with frequent vaginal yeast infections that they\ncould try eating 6 ounces of yogurt daily. It turns out most would\nrather just use anti-fungal creams when they get yeast infections.\n\n>yogurt dangerous). If this were a standard part of medical practice, as \n>Gordon R. says it is, then the incidence of GI distress and vaginal yeast \n>infections should decline.\n\nAgain, this just isn\'t what the systemic yeast syndrome is about, and\nhas nothing to do with the quack therapies that were being discussed.\nThere is some evidence that attempts to reinoculate the GI tract with\nbacteria after antibiotic therapy don\'t seem to be very helpful in\nreducing diarrhea, but I don\'t think anyone would view this as a\nquack therapy.\n-- \nDavid Rind\[email protected]\n']
labels = [2.0, 2.0]
sc = SeldonClient(
gateway="ambassador",
gateway_endpoint=API_AMBASSADOR,
deployment_name="seldon-model-server",
payload_type="ndarray",
namespace="seldon",
transport="rest")
sm_result = sc.predict(data=np.array(data))
logging.info(sm_result)
result = seldon_message_to_json(sm_result.response)
logging.info(result)
values = result.get("data", {}).get("ndarray", {})
assert (values == labels)
def handle_raw_custom_metrics(
msg: Union[prediction_pb2.SeldonMessage, Dict],
seldon_metrics: SeldonMetrics,
is_proto: bool,
):
"""
Update SeldonMetrics object with custom metrics from raw methods.
If INCLUDE_METRICS_IN_CLIENT_RESPONSE environmental variable is set to "true"
metrics will be dropped from msg.
"""
if is_proto:
metrics = seldon_message_to_json(msg.meta).get("metrics", [])
if metrics and not INCLUDE_METRICS_IN_CLIENT_RESPONSE:
del msg.meta.metrics[:]
else:
metrics = msg.get("meta", {}).get("metrics", [])
if metrics and not INCLUDE_METRICS_IN_CLIENT_RESPONSE:
del msg["meta"]["metrics"]
seldon_metrics.update(metrics)
def aggregate(
user_model: Any,
request: Union[prediction_pb2.SeldonMessageList, List, Dict],
seldon_metrics: SeldonMetrics,
) -> Union[prediction_pb2.SeldonMessage, List, Dict]:
"""
Aggregate a list of payloads
Parameters
----------
user_model
A Seldon user model
request
SeldonMessage proto
Returns
-------
Aggregated SeldonMessage proto
"""
def merge_meta(meta_list):
tags = {}
for meta in meta_list:
if meta:
tags.update(meta.get("tags", {}))
return {"tags": tags}
def merge_metrics(meta_list, custom_metrics):
metrics = []
for meta in meta_list:
if meta:
metrics.extend(meta.get("metrics", []))
metrics.extend(custom_metrics)
return metrics
is_proto = isinstance(request, prediction_pb2.SeldonMessageList)
if hasattr(user_model, "aggregate_rest"):
logger.warning(
"aggregate_rest is deprecated. Please use aggregate_raw")
return user_model.aggregate_rest(request)
elif hasattr(user_model, "aggregate_grpc"):
logger.warning(
"aggregate_grpc is deprecated. Please use aggregate_raw")
return user_model.aggregate_grpc(request)
else:
if hasattr(user_model, "aggregate_raw"):
try:
response = user_model.aggregate_raw(request)
if is_proto:
metrics = seldon_message_to_json(response.meta).get(
"metrics", [])
else:
metrics = response.get("meta", {}).get("metrics", [])
seldon_metrics.update(metrics)
return response
except SeldonNotImplementedError:
pass
if is_proto:
features_list = []
names_list = []
meta_list = []
for msg in request.seldonMessages:
(features, meta, datadef,
data_type) = extract_request_parts(msg)
features_list.append(features)
names_list.append(datadef.names)
meta_list.append(meta)
client_response = client_aggregate(user_model, features_list,
names_list)
metrics = client_custom_metrics(user_model)
if seldon_metrics is not None:
seldon_metrics.update(metrics)
return construct_response(
user_model,
False,
request.seldonMessages[0],
client_response,
merge_meta(meta_list),
merge_metrics(meta_list, metrics),
)
else:
features_list = []
names_list = []
if isinstance(request, list):
msgs = request
elif "seldonMessages" in request and isinstance(
request["seldonMessages"], list):
msgs = request["seldonMessages"]
else:
raise SeldonMicroserviceException(
f"Invalid request data type: {request}")
meta_list = []
for msg in msgs:
(features, meta, datadef,
data_type) = extract_request_parts_json(msg)
class_names = datadef[
"names"] if datadef and "names" in datadef else []
features_list.append(features)
names_list.append(class_names)
meta_list.append(meta)
client_response = client_aggregate(user_model, features_list,
names_list)
metrics = client_custom_metrics(user_model)
if seldon_metrics is not None:
seldon_metrics.update(metrics)
return construct_response_json(
user_model,
False,
msgs[0],
client_response,
merge_meta(meta_list),
merge_metrics(meta_list, metrics),
)
def mocked_requests_post_success_json_data(url, *args, **kwargs):
request = json_to_seldon_message({"jsonData": JSON_TEST_DATA})
json = seldon_message_to_json(request)
return MockResponse(json, 200, text="{}")
def mocked_requests_post_success(url, *args, **kwargs):
data = np.random.rand(1, 1)
datadef = array_to_grpc_datadef("tensor", data)
request = prediction_pb2.SeldonMessage(data=datadef)
json = seldon_message_to_json(request)
return MockResponse(json, 200, text="{}")
def route(
user_model: Any,
request: Union[prediction_pb2.SeldonMessage, List, Dict],
seldon_metrics: SeldonMetrics,
) -> Union[prediction_pb2.SeldonMessage, List, Dict]:
"""
Parameters
----------
user_model
A Seldon user model
request
A SelodonMessage proto
Returns
-------
"""
is_proto = isinstance(request, prediction_pb2.SeldonMessage)
if hasattr(user_model, "route_rest"):
logger.warning("route_rest is deprecated. Please use route_raw")
return user_model.route_rest(request)
elif hasattr(user_model, "route_grpc"):
logger.warning("route_grpc is deprecated. Please use route_raw")
return user_model.route_grpc(request)
else:
if hasattr(user_model, "route_raw"):
try:
response = user_model.route_raw(request)
if is_proto:
metrics = seldon_message_to_json(response.meta).get(
"metrics", [])
else:
metrics = response.get("meta", {}).get("metrics", [])
seldon_metrics.update(metrics)
return response
except SeldonNotImplementedError:
pass
if is_proto:
(features, meta, datadef,
data_type) = extract_request_parts(request)
client_response = client_route(user_model,
features,
datadef.names,
meta=meta)
if not isinstance(client_response, int):
raise SeldonMicroserviceException(
"Routing response must be int but got " +
str(client_response))
client_response_arr = np.array([[client_response]])
metrics = client_custom_metrics(user_model)
if seldon_metrics is not None:
seldon_metrics.update(metrics)
return construct_response(user_model, False, request,
client_response_arr, None, metrics)
else:
(features, meta, datadef,
data_type) = extract_request_parts_json(request)
class_names = datadef[
"names"] if datadef and "names" in datadef else []
client_response = client_route(user_model,
features,
class_names,
meta=meta)
if not isinstance(client_response, int):
raise SeldonMicroserviceException(
"Routing response must be int but got " +
str(client_response))
client_response_arr = np.array([[client_response]])
metrics = client_custom_metrics(user_model)
if seldon_metrics is not None:
seldon_metrics.update(metrics)
return construct_response_json(user_model, False, request,
client_response_arr, None, metrics)
def predict(
user_model: Any,
request: Union[prediction_pb2.SeldonMessage, List, Dict],
seldon_metrics: SeldonMetrics,
) -> Union[prediction_pb2.SeldonMessage, List, Dict]:
"""
Call the user model to get a prediction and package the response
Parameters
----------
user_model
User defined class instance
request
The incoming request
Returns
-------
The prediction
"""
is_proto = isinstance(request, prediction_pb2.SeldonMessage)
if hasattr(user_model, "predict_rest") and not is_proto:
logger.warning("predict_rest is deprecated. Please use predict_raw")
return user_model.predict_rest(request)
elif hasattr(user_model, "predict_grpc") and is_proto:
logger.warning("predict_grpc is deprecated. Please use predict_raw")
return user_model.predict_grpc(request)
else:
if hasattr(user_model, "predict_raw"):
try:
response = user_model.predict_raw(request)
if is_proto:
metrics = seldon_message_to_json(response.meta).get(
"metrics", [])
else:
metrics = response.get("meta", {}).get("metrics", [])
seldon_metrics.update(metrics)
return response
except SeldonNotImplementedError:
pass
if is_proto:
(features, meta, datadef,
data_type) = extract_request_parts(request)
client_response = client_predict(user_model,
features,
datadef.names,
meta=meta)
metrics = client_custom_metrics(user_model)
if seldon_metrics is not None:
seldon_metrics.update(metrics)
return construct_response(user_model, False, request,
client_response, meta, metrics)
else:
(features, meta, datadef,
data_type) = extract_request_parts_json(request)
class_names = datadef[
"names"] if datadef and "names" in datadef else []
client_response = client_predict(user_model,
features,
class_names,
meta=meta)
metrics = client_custom_metrics(user_model)
if seldon_metrics is not None:
seldon_metrics.update(metrics)
return construct_response_json(user_model, False, request,
client_response, meta, metrics)
