def test_proto_seldon_runtime_data_transform_output(cls, client_gets_metrics):
user_object = cls()
seldon_metrics = SeldonMetrics()
app = SeldonModelGRPC(user_object, seldon_metrics)
datadef = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=(2, 1), values=np.array([1, 2]))
)
request = prediction_pb2.SeldonMessage(data=datadef)
resp = app.TransformOutput(request, None)
j = json.loads(json_format.MessageToJson(resp))
assert j["data"] == {
"names": ["t:0"],
"tensor": {"shape": [2, 1], "values": [1.0, 2.0]},
}
assert j["meta"]["tags"] == EXPECTED_TAGS
assert ("metrics" in j["meta"]) == client_gets_metrics
data = seldon_metrics.data[os.getpid()]
verify_seldon_metrics(data, 1, [0.0202], OUTPUT_TRANSFORM_METRIC_METHOD_TAG)
resp = app.TransformOutput(request, None)
j = json.loads(json_format.MessageToJson(resp))
assert j["data"] == {
"names": ["t:0"],
"tensor": {"shape": [2, 1], "values": [1.0, 2.0]},
}
assert j["meta"]["tags"] == EXPECTED_TAGS
assert ("metrics" in j["meta"]) == client_gets_metrics
data = seldon_metrics.data[os.getpid()]
verify_seldon_metrics(data, 2, [0.0202, 0.0202], OUTPUT_TRANSFORM_METRIC_METHOD_TAG)
def test_proto_seldon_metrics_transform_output(cls):
user_object = cls()
seldon_metrics = SeldonMetrics()
app = SeldonModelGRPC(user_object, seldon_metrics)
datadef = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=(2, 1), values=np.array([1, 2])))
request = prediction_pb2.SeldonMessage(data=datadef)
app.TransformOutput(request, None)
data = seldon_metrics.data[os.getpid()]
assert data["GAUGE", "mygauge"]["value"] == 100
assert data["GAUGE", "customtag"]["value"] == 200
assert data["GAUGE", "customtag"]["tags"] == {"mytag": "mytagvalue"}
assert data["COUNTER", "mycounter"]["value"] == 1
assert np.allclose(
np.histogram([20.2 / 1000], BINS)[0], data["TIMER",
"mytimer"]["value"][0])
assert np.allclose(data["TIMER", "mytimer"]["value"][1], 0.0202)
app.TransformOutput(request, None)
data = seldon_metrics.data[os.getpid()]
assert data["GAUGE", "mygauge"]["value"] == 100
assert data["GAUGE", "customtag"]["value"] == 200
assert data["GAUGE", "customtag"]["tags"] == {"mytag": "mytagvalue"}
assert data["COUNTER", "mycounter"]["value"] == 2
assert np.allclose(
np.histogram([20.2 / 1000, 20.2 / 1000], BINS)[0],
data["TIMER", "mytimer"]["value"][0],
)
assert np.allclose(data["TIMER", "mytimer"]["value"][1], 0.0404)
def test_transform_proto_output_passes_through_metrics():
user_object = UserObject()
seldon_metrics = SeldonMetrics()
app = SeldonModelGRPC(user_object, seldon_metrics)
arr = np.array([1, 2])
datadef = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr))
meta = prediction_pb2.Meta()
json_format.ParseDict(
{"metrics": [{
"key": "request_gauge",
"type": "GAUGE",
"value": 100
}]}, meta)
request = prediction_pb2.SeldonMessage(data=datadef, meta=meta)
resp = app.TransformOutput(request, None)
jStr = json_format.MessageToJson(resp)
j = json.loads(jStr)
logging.info(j)
assert j["meta"]["metrics"][0]["key"] == "request_gauge"
assert j["meta"]["metrics"][0]["value"] == 100
assert j["meta"]["metrics"][1]["key"] == user_object.metrics()[0]["key"]
assert j["meta"]["metrics"][1]["value"] == user_object.metrics(
)[0]["value"]
assert j["data"]["tensor"]["shape"] == [2, 1]
assert j["data"]["tensor"]["values"] == [1, 2]
def test_transform_output_proto_bin_data():
user_object = UserObject()
app = SeldonModelGRPC(user_object)
binData = b"\0\1"
request = prediction_pb2.SeldonMessage(binData=binData)
resp = app.TransformOutput(request, None)
assert resp.binData == binData
def test_transform_output_proto_bin_data_nparray():
user_object = UserObject(ret_nparray=True)
app = SeldonModelGRPC(user_object)
binData = b"\0\1"
request = prediction_pb2.SeldonMessage(binData=binData)
resp = app.TransformOutput(request, None)
jStr = json_format.MessageToJson(resp)
j = json.loads(jStr)
print(j)
assert j["data"]["tensor"]["values"] == list(user_object.nparray.flatten())
def test_proto_seldon_metrics_transform_output(cls, client_gets_metrics):
user_object = cls()
seldon_metrics = SeldonMetrics()
app = SeldonModelGRPC(user_object, seldon_metrics)
datadef = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=(2, 1), values=np.array([1, 2])))
request = prediction_pb2.SeldonMessage(data=datadef)
resp = app.TransformOutput(request, None)
assert ("metrics" in json.loads(
json_format.MessageToJson(resp))["meta"]) == client_gets_metrics
data = seldon_metrics.data[os.getpid()]
verify_seldon_metrics(data, 1, [0.0202])
resp = app.TransformOutput(request, None)
assert ("metrics" in json.loads(
json_format.MessageToJson(resp))["meta"]) == client_gets_metrics
data = seldon_metrics.data[os.getpid()]
verify_seldon_metrics(data, 2, [0.0202, 0.0202])
def test_transform_output_proto_lowlevel_ok():
user_object = UserObjectLowLevelGrpc()
app = SeldonModelGRPC(user_object)
arr = np.array([1, 2])
datadef = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr))
request = prediction_pb2.SeldonMessage(data=datadef)
resp = app.TransformOutput(request, None)
jStr = json_format.MessageToJson(resp)
j = json.loads(jStr)
print(j)
assert j["data"]["tensor"]["shape"] == [2, 1]
assert j["data"]["tensor"]["values"] == [9, 9]
def test_transform_output_proto_ok():
user_object = UserObject()
app = SeldonModelGRPC(user_object)
arr = np.array([1, 2])
datadef = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr))
request = prediction_pb2.SeldonMessage(data=datadef)
resp = app.TransformOutput(request, None)
jStr = json_format.MessageToJson(resp)
j = json.loads(jStr)
print(j)
assert j["meta"]["tags"] == {"mytag": 1}
# add default type
assert j["meta"]["metrics"][0]["key"] == user_object.metrics()[0]["key"]
assert j["meta"]["metrics"][0]["value"] == user_object.metrics(
)[0]["value"]
assert j["data"]["tensor"]["shape"] == [2, 1]
assert j["data"]["tensor"]["values"] == [1, 2]
def test_transform_proto_output_passes_through_tags():
user_object = UserObject()
app = SeldonModelGRPC(user_object)
arr = np.array([1, 2])
datadef = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr)
)
meta = prediction_pb2.Meta()
json_format.ParseDict({"tags": {"foo": "bar"}}, meta)
request = prediction_pb2.SeldonMessage(data=datadef, meta=meta)
resp = app.TransformOutput(request, None)
jStr = json_format.MessageToJson(resp)
j = json.loads(jStr)
logging.info(j)
assert j["meta"]["tags"] == {"foo": "bar", "mytag": 1}
assert j["meta"]["metrics"][0]["key"] == user_object.metrics()[0]["key"]
assert j["meta"]["metrics"][0]["value"] == user_object.metrics()[0]["value"]
assert j["data"]["tensor"]["shape"] == [2, 1]
assert j["data"]["tensor"]["values"] == [1, 2]
def test_transform_output_proto_gets_meta():
user_object = UserObject(ret_meta=True)
app = SeldonModelGRPC(user_object)
arr = np.array([1, 2])
datadef = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr))
meta = prediction_pb2.Meta()
metaJson = {"puid": "abc"}
json_format.ParseDict(metaJson, meta)
request = prediction_pb2.SeldonMessage(data=datadef, meta=meta)
resp = app.TransformOutput(request, None)
jStr = json_format.MessageToJson(resp)
j = json.loads(jStr)
print(j)
assert j["meta"]["tags"] == {"inc_meta": {"puid": "abc"}}
# add default type
assert j["meta"]["metrics"][0]["key"] == user_object.metrics()[0]["key"]
assert j["meta"]["metrics"][0]["value"] == user_object.metrics(
)[0]["value"]
assert j["data"]["tensor"]["shape"] == [2, 1]
assert j["data"]["tensor"]["values"] == [1, 2]