def test_aggregate_proto_combines_metrics():
user_object = UserObject()
seldon_metrics = SeldonMetrics()
app = SeldonModelGRPC(user_object, seldon_metrics)
arr1 = np.array([1, 2])
meta1 = prediction_pb2.Meta()
json_format.ParseDict(
{
"metrics": [{
"key": "request_gauge_1",
"type": "GAUGE",
"value": 100
}]
}, meta1)
datadef1 = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr1))
arr2 = np.array([3, 4])
meta2 = prediction_pb2.Meta()
json_format.ParseDict(
{
"metrics": [{
"key": "request_gauge_2",
"type": "GAUGE",
"value": 200
}]
}, meta2)
datadef2 = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr2))
msg1 = prediction_pb2.SeldonMessage(data=datadef1, meta=meta1)
msg2 = prediction_pb2.SeldonMessage(data=datadef2, meta=meta2)
request = prediction_pb2.SeldonMessageList(seldonMessages=[msg1, msg2])
resp = app.Aggregate(request, None)
jStr = json_format.MessageToJson(resp)
j = json.loads(jStr)
logging.info(j)
assert j["meta"]["tags"] == {"mytag": 1}
assert j["meta"]["metrics"][0]["key"] == "request_gauge_1"
assert j["meta"]["metrics"][0]["value"] == 100
assert j["meta"]["metrics"][1]["key"] == "request_gauge_2"
assert j["meta"]["metrics"][1]["value"] == 200
assert j["meta"]["metrics"][2]["key"] == user_object.metrics()[0]["key"]
assert j["meta"]["metrics"][2]["value"] == user_object.metrics(
)[0]["value"]
assert j["data"]["tensor"]["shape"] == [2, 1]
assert j["data"]["tensor"]["values"] == [1, 2]
def Route(self, request, context):
if hasattr(self.user_model, "route_grpc"):
return self.user_model.route_grpc(request)
else:
datadef = request.data
features = grpc_datadef_to_array(datadef)
routing = np.array([[route(self.user_model, features, datadef.names)]])
# TODO: check that predictions is 2 dimensional
class_names = []
data = array_to_grpc_datadef(
routing, class_names, request.data.WhichOneof("data_oneof"))
# Construct meta data
meta = prediction_pb2.Meta()
metaJson = {}
tags = get_custom_tags(self.user_model)
if tags:
metaJson["tags"] = tags
metrics = get_custom_metrics(self.user_model)
if metrics:
metaJson["metrics"] = metrics
json_format.ParseDict(metaJson, meta)
return prediction_pb2.SeldonMessage(data=data, meta=meta)
def test_transform_proto_output_passes_through_metrics():
user_object = UserObject()
seldon_metrics = SeldonMetrics()
app = SeldonModelGRPC(user_object, seldon_metrics)
arr = np.array([1, 2])
datadef = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr))
meta = prediction_pb2.Meta()
json_format.ParseDict(
{"metrics": [{
"key": "request_gauge",
"type": "GAUGE",
"value": 100
}]}, meta)
request = prediction_pb2.SeldonMessage(data=datadef, meta=meta)
resp = app.TransformOutput(request, None)
jStr = json_format.MessageToJson(resp)
j = json.loads(jStr)
logging.info(j)
assert j["meta"]["metrics"][0]["key"] == "request_gauge"
assert j["meta"]["metrics"][0]["value"] == 100
assert j["meta"]["metrics"][1]["key"] == user_object.metrics()[0]["key"]
assert j["meta"]["metrics"][1]["value"] == user_object.metrics(
)[0]["value"]
assert j["data"]["tensor"]["shape"] == [2, 1]
assert j["data"]["tensor"]["values"] == [1, 2]
def construct_response(user_model: SeldonComponent, is_request: bool, client_request: prediction_pb2.SeldonMessage,
client_raw_response: Union[np.ndarray, str, bytes, dict]) -> prediction_pb2.SeldonMessage:
"""
Parameters
----------
user_model
Client user class
is_request
Whether this is part of the request flow as opposed to the response flow
client_request
The request received
client_raw_response
The raw client response from their model
Returns
-------
A SeldonMessage proto response
"""
data_type = client_request.WhichOneof("data_oneof")
meta = prediction_pb2.Meta()
meta_json: Dict = {}
tags = client_custom_tags(user_model)
if tags:
meta_json["tags"] = tags
metrics = client_custom_metrics(user_model)
if metrics:
meta_json["metrics"] = metrics
if client_request.meta:
if client_request.meta.puid:
meta_json["puid"] = client_request.meta.puid
json_format.ParseDict(meta_json, meta)
if isinstance(client_raw_response, np.ndarray) or isinstance(client_raw_response, list):
client_raw_response = np.array(client_raw_response)
if is_request:
names = client_feature_names(user_model, client_request.data.names)
else:
names = client_class_names(user_model, client_raw_response)
if data_type == "data": # If request is using defaultdata then return what was sent if is numeric response else ndarray
if np.issubdtype(client_raw_response.dtype, np.number):
default_data_type = client_request.data.WhichOneof("data_oneof")
else:
default_data_type = "ndarray"
else: # If numeric response return as tensor else return as ndarray
if np.issubdtype(client_raw_response.dtype, np.number):
default_data_type = "tensor"
else:
default_data_type = "ndarray"
data = array_to_grpc_datadef(default_data_type, client_raw_response, names)
return prediction_pb2.SeldonMessage(data=data, meta=meta)
elif isinstance(client_raw_response, str):
return prediction_pb2.SeldonMessage(strData=client_raw_response, meta=meta)
elif isinstance(client_raw_response, dict):
jsonDataResponse = ParseDict(client_raw_response, prediction_pb2.SeldonMessage().jsonData)
return prediction_pb2.SeldonMessage(jsonData=jsonDataResponse, meta=meta)
elif isinstance(client_raw_response, (bytes, bytearray)):
return prediction_pb2.SeldonMessage(binData=client_raw_response, meta=meta)
else:
raise SeldonMicroserviceException("Unknown data type returned as payload:" + client_raw_response)
def test_proto_tags():
metric = {
"tags": {
"t1": "t2"
},
"metrics": [
{
"type": "COUNTER",
"key": "mycounter",
"value": 1.2
},
{
"type": "GAUGE",
"key": "mygauge",
"value": 1.2
},
{
"type": "TIMER",
"key": "mytimer",
"value": 1.2
},
],
}
meta = prediction_pb2.Meta()
json_format.ParseDict(metric, meta)
jStr = json_format.MessageToJson(meta)
j = json.loads(jStr)
assert "mycounter" == j["metrics"][0]["key"]
assert 1.2 == pytest.approx(j["metrics"][0]["value"], 0.01)
assert "GAUGE" == j["metrics"][1]["type"]
assert "mygauge" == j["metrics"][1]["key"]
assert 1.2 == pytest.approx(j["metrics"][1]["value"], 0.01)
assert "TIMER" == j["metrics"][2]["type"]
assert "mytimer" == j["metrics"][2]["key"]
assert 1.2 == pytest.approx(j["metrics"][2]["value"], 0.01)
def test_model_template_app_grpc_metrics(tracing):
with start_microservice(join(dirname(__file__), "model-template-app"),
tracing=tracing,
grpc=True):
data = np.array([[1, 2]])
datadef = prediction_pb2.DefaultData(tensor=prediction_pb2.Tensor(
shape=data.shape, values=data.flatten()))
meta = prediction_pb2.Meta()
json_format.ParseDict(
{"metrics": [{
"key": "mygauge",
"type": "GAUGE",
"value": 100
}]}, meta)
request = prediction_pb2.SeldonMessage(data=datadef, meta=meta)
channel = grpc.insecure_channel("0.0.0.0:5000")
stub = prediction_pb2_grpc.ModelStub(channel)
response = retry_method(stub.Predict, kwargs=dict(request=request))
assert response.data.tensor.shape[0] == 1
assert response.data.tensor.shape[1] == 2
assert response.data.tensor.values[0] == 1
assert response.data.tensor.values[1] == 2
assert response.meta.metrics[0].key == "mygauge"
assert response.meta.metrics[0].value == 100
def TransformOutput(self, request, context):
if hasattr(self.user_model, "transform_output_grpc"):
return self.user_model.transform_output_grpc(request)
else:
features = get_data_from_proto(request)
datadef = request.data
data_type = request.WhichOneof("data_oneof")
# Construct meta data
meta = prediction_pb2.Meta()
metaJson = {}
tags = get_custom_tags(self.user_model)
if tags:
metaJson["tags"] = tags
metrics = get_custom_metrics(self.user_model)
if metrics:
metaJson["metrics"] = metrics
json_format.ParseDict(metaJson, meta)
transformed = transform_output(self.user_model, features,
datadef.names)
if isinstance(transformed, np.ndarray) or data_type == "data":
transformed = np.array(transformed)
class_names = get_class_names(self.user_model, datadef.names)
if data_type == "data":
default_data_type = request.data.WhichOneof("data_oneof")
else:
default_data_type = "tensor"
data = array_to_grpc_datadef(transformed, class_names,
default_data_type)
return prediction_pb2.SeldonMessage(data=data, meta=meta)
else:
return prediction_pb2.SeldonMessage(binData=transformed,
meta=meta)
def route_raw(self, msg):
logging.info("Route raw called")
meta = prediction_pb2.Meta()
json_format.ParseDict({"metrics": self._metrics}, meta)
arr = np.array([22])
datadef = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=(1, 1), values=arr))
request = prediction_pb2.SeldonMessage(data=datadef, meta=meta)
return request
def test_aggregate_proto_combines_tags():
user_object = UserObject()
seldon_metrics = SeldonMetrics()
app = SeldonModelGRPC(user_object, seldon_metrics)
arr1 = np.array([1, 2])
meta1 = prediction_pb2.Meta()
json_format.ParseDict({"tags": {"input-1": "yes", "common": 1}}, meta1)
datadef1 = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr1)
)
arr2 = np.array([3, 4])
meta2 = prediction_pb2.Meta()
json_format.ParseDict({"tags": {"input-2": "yes", "common": 2}}, meta2)
datadef2 = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr2)
)
msg1 = prediction_pb2.SeldonMessage(data=datadef1, meta=meta1)
msg2 = prediction_pb2.SeldonMessage(data=datadef2, meta=meta2)
request = prediction_pb2.SeldonMessageList(seldonMessages=[msg1, msg2])
resp = app.Aggregate(request, None)
jStr = json_format.MessageToJson(resp)
j = json.loads(jStr)
logging.info(j)
assert j["meta"]["tags"] == {
"common": 2,
"input-1": "yes",
"input-2": "yes",
"mytag": 1,
}
# add default type
assert j["meta"]["metrics"][0]["key"] == user_object.metrics()[0]["key"]
assert j["meta"]["metrics"][0]["value"] == user_object.metrics()[0]["value"]
assert j["data"]["tensor"]["shape"] == [2, 1]
assert j["data"]["tensor"]["values"] == [1, 2]
def test_proto_requestPath_ok():
user_object = UserObject()
seldon_metrics = SeldonMetrics()
app = SeldonModelGRPC(user_object, seldon_metrics)
arr = np.array([1, 2])
datadef = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr))
meta = prediction_pb2.Meta()
json_format.ParseDict({"tags": {"foo": "bar"}}, meta)
request = prediction_pb2.SeldonMessage(data=datadef, meta=meta)
resp = app.Predict(request, None)
jStr = json_format.MessageToJson(resp)
j = json.loads(jStr)
logging.info(j)
assert j["meta"]["requestPath"] == {"my-test-model": "my-test-model-image"}
def test_proto_feedback():
user_object = UserObject()
app = SeldonModelGRPC(user_object)
arr = np.array([1, 2])
datadef = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr))
meta = prediction_pb2.Meta()
metaJson = {}
routing = {"1": 1}
metaJson["routing"] = routing
json_format.ParseDict(metaJson, meta)
request = prediction_pb2.SeldonMessage(data=datadef)
response = prediction_pb2.SeldonMessage(meta=meta, data=datadef)
feedback = prediction_pb2.Feedback(request=request,
response=response,
reward=1.0)
resp = app.SendFeedback(feedback, None)
def test_model_template_app_grpc_tags(microservice):
data = np.array([[1, 2]])
datadef = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=data.shape, values=data.flatten()))
meta = prediction_pb2.Meta()
json_format.ParseDict({"tags": {"foo": "bar"}}, meta)
request = prediction_pb2.SeldonMessage(data=datadef, meta=meta)
channel = grpc.insecure_channel("0.0.0.0:5000")
stub = prediction_pb2_grpc.ModelStub(channel)
response = retry_method(stub.Predict, kwargs=dict(request=request))
assert response.data.tensor.shape[0] == 1
assert response.data.tensor.shape[1] == 2
assert response.data.tensor.values[0] == 1
assert response.data.tensor.values[1] == 2
assert response.meta.tags["foo"].string_value == "bar"
def test_proto_passes_through_tags():
user_object = UserObject()
app = SeldonModelGRPC(user_object)
arr = np.array([1, 2])
datadef = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr))
meta = prediction_pb2.Meta()
json_format.ParseDict({"tags": {"foo": "bar"}}, meta)
request = prediction_pb2.SeldonMessage(data=datadef, meta=meta)
resp = app.Predict(request, None)
jStr = json_format.MessageToJson(resp)
j = json.loads(jStr)
logging.info(j)
assert j["meta"]["tags"] == {"foo": "bar", "mytag": 1}
assert j["meta"]["metrics"][0]["key"] == user_object.metrics()[0]["key"]
assert j["meta"]["metrics"][0]["value"] == user_object.metrics(
)[0]["value"]
assert j["data"]["tensor"]["shape"] == [2, 1]
assert j["data"]["tensor"]["values"] == [1, 2]
def Predict(self, request, context):
if hasattr(self.user_model, "predict_grpc"):
return self.user_model.predict_grpc(request)
else:
features = get_data_from_proto(request)
meta = get_meta_from_proto(request)
datadef = request.data
data_type = request.WhichOneof("data_oneof")
predictions = predict(self.user_model,
features,
datadef.names,
meta=meta)
# Construct meta data
meta = prediction_pb2.Meta()
metaJson = {}
tags = get_custom_tags(self.user_model)
if tags:
metaJson["tags"] = tags
metrics = get_custom_metrics(self.user_model)
if metrics:
metaJson["metrics"] = metrics
json_format.ParseDict(metaJson, meta)
if isinstance(predictions, np.ndarray) or data_type == "data":
predictions = np.array(predictions)
if len(predictions.shape) > 1:
class_names = get_class_names(self.user_model,
predictions.shape[1])
else:
class_names = []
if data_type == "data":
default_data_type = request.data.WhichOneof("data_oneof")
else:
default_data_type = "tensor"
data = array_to_grpc_datadef(predictions, class_names,
default_data_type)
return prediction_pb2.SeldonMessage(data=data, meta=meta)
else:
return prediction_pb2.SeldonMessage(binData=predictions,
meta=meta)
def test_proto_gets_meta():
user_object = UserObject(ret_meta=True)
app = SeldonModelGRPC(user_object)
arr = np.array([1, 2])
datadef = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr))
meta = prediction_pb2.Meta()
metaJson = {"puid": "abc"}
json_format.ParseDict(metaJson, meta)
request = prediction_pb2.SeldonMessage(data=datadef, meta=meta)
resp = app.Predict(request, None)
jStr = json_format.MessageToJson(resp)
j = json.loads(jStr)
print(j)
assert j["meta"]["tags"] == {"inc_meta": {"puid": "abc"}}
assert j["meta"]["metrics"][0]["key"] == user_object.metrics()[0]["key"]
assert j["meta"]["metrics"][0]["value"] == user_object.metrics(
)[0]["value"]
assert j["data"]["tensor"]["shape"] == [2, 1]
assert j["data"]["tensor"]["values"] == [1, 2]
def Aggregate(self, request, context):
if hasattr(self.user_model, "aggregate_grpc"):
return self.user_model.aggregate_grpc(request)
else:
features_list = []
names_list = []
for msg in request.seldonMessages:
features = get_data_from_proto(msg)
features_list.append(features)
names_list.append(msg.data.names)
data_type = request.seldonMessages[0].WhichOneof("data_oneof")
aggregated = aggregate(self.user_model, features_list, names_list)
# Construct meta data
meta = prediction_pb2.Meta()
metaJson = {}
tags = get_custom_tags(self.user_model)
if tags:
metaJson["tags"] = tags
metrics = get_custom_metrics(self.user_model)
if metrics:
metaJson["metrics"] = metrics
json_format.ParseDict(metaJson, meta)
if isinstance(aggregated, np.ndarray) or data_type == "data":
aggregated = np.array(aggregated)
feature_names = get_feature_names(self.user_model, [])
if data_type == "data":
default_data_type = request.seldonMessages[
0].data.WhichOneof("data_oneof")
else:
default_data_type = "tensor"
data = array_to_grpc_datadef(aggregated, feature_names,
default_data_type)
return prediction_pb2.SeldonMessage(data=data, meta=meta)
else:
return prediction_pb2.SeldonMessage(binData=aggregated,
meta=meta)
def test_model_template_app_grpc_tags(tracing):
with start_microservice(join(dirname(__file__), "model-template-app"),
tracing=tracing,
grpc=True):
data = np.array([[1, 2]])
datadef = prediction_pb2.DefaultData(tensor=prediction_pb2.Tensor(
shape=data.shape, values=data.flatten()))
meta = prediction_pb2.Meta()
json_format.ParseDict({"tags": {"foo": "bar"}}, meta)
request = prediction_pb2.SeldonMessage(data=datadef, meta=meta)
channel = grpc.insecure_channel("0.0.0.0:5000")
stub = prediction_pb2_grpc.ModelStub(channel)
response = stub.Predict(request=request)
assert response.data.tensor.shape[0] == 1
assert response.data.tensor.shape[1] == 2
assert response.data.tensor.values[0] == 1
assert response.data.tensor.values[1] == 2
assert response.meta.tags["foo"].string_value == "bar"
def test_proto_metrics():
metrics = [{"type": "COUNTER", "key": "a", "value": 1}]
meta = prediction_pb2.Meta()
for metric in metrics:
mpb2 = meta.metrics.add()
json_format.ParseDict(metric, mpb2)