def test_model_health_status():
user_object = UserObject()
seldon_metrics = SeldonMetrics()
app = get_rest_microservice(user_object, seldon_metrics)
client = app.test_client()
rv = client.get(HEALTH_STATUS_URL)
assert rv.status_code == 200
j = json.loads(rv.data)
logging.info(j)
assert j["data"]["tensor"]["values"] == UserObject.HEALTH_STATUS_REPONSE
def test_transformer_output_lowlevel_raw_inherited_ok():
user_object = UserObjectLowLevelRawInherited()
seldon_metrics = SeldonMetrics()
app = get_rest_microservice(user_object, seldon_metrics)
client = app.test_client()
rv = client.get('/transform-output?json={"data":{"ndarray":[1]}}')
j = json.loads(rv.data)
logging.info(j)
assert rv.status_code == 200
assert j["data"]["tensor"]["values"] == [9, 9]
def test_model_non200status_lowlevel():
user_object = UserObjectLowLevelWithStatusInResponse()
seldon_metrics = SeldonMetrics()
app = get_rest_microservice(user_object, seldon_metrics)
client = app.test_client()
rv = client.get(
'/predict?json={"request":{"data":{"ndarray":[]}},"reward":1.0}')
j = json.loads(rv.data)
logging.info(j)
assert rv.status_code == 400
def test_model_health_status_raw():
user_object = UserObjectLowLevel()
seldon_metrics = SeldonMetrics()
app = get_rest_microservice(user_object, seldon_metrics)
client = app.test_client()
rv = client.get(HEALTH_STATUS_URL)
assert rv.status_code == 200
j = json.loads(rv.data)
assert j["data"][
"ndarray"] == UserObjectLowLevel.HEALTH_STATUS_RAW_RESPONSE
def test_model_no_json():
user_object = UserObject()
seldon_metrics = SeldonMetrics()
app = get_rest_microservice(user_object, seldon_metrics)
client = app.test_client()
uo = UserObject()
rv = client.get("/predict?")
j = json.loads(rv.data)
logging.info(j)
assert rv.status_code == 400
def test_model_lowlevel_ok():
user_object = UserObjectLowLevel()
seldon_metrics = SeldonMetrics()
app = get_rest_microservice(user_object, seldon_metrics)
client = app.test_client()
rv = client.get('/predict?json={"data":{"ndarray":[1,2]}}')
j = json.loads(rv.data)
logging.info(j)
assert rv.status_code == 200
assert j["data"]["ndarray"] == [9, 9]
def test_model_feedback_lowlevel_ok():
user_object = UserObjectLowLevel()
seldon_metrics = SeldonMetrics()
app = get_rest_microservice(user_object, seldon_metrics)
client = app.test_client()
rv = client.get(
'/send-feedback?json={"request":{"data":{"ndarray":[]}},"reward":1.0}')
j = json.loads(rv.data)
logging.info(j)
assert rv.status_code == 200
def test_model_metadata():
user_object = UserObject()
seldon_metrics = SeldonMetrics()
app = get_rest_microservice(user_object, seldon_metrics)
client = app.test_client()
rv = client.get(METADATA_URL)
assert rv.status_code == 200
j = json.loads(rv.data)
logging.info(j)
assert j == UserObject.METADATA_RESPONSE
def test_aggreate_invalid_message():
user_object = UserObject()
seldon_metrics = SeldonMetrics()
app = get_rest_microservice(user_object, seldon_metrics)
client = app.test_client()
rv = client.get('/aggregate?json={"wrong":[{"data":{"ndarray":[1]}}]}')
assert rv.status_code == 400
j = json.loads(rv.data)
logging.info(j)
assert j["status"]["reason"] == "MICROSERVICE_BAD_DATA"
def test_proto_feedback_custom():
user_object = UserObjectLowLevel()
seldon_metrics = SeldonMetrics()
app = SeldonModelGRPC(user_object, seldon_metrics)
arr = np.array([1, 2])
datadef = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr))
request = prediction_pb2.SeldonMessage(data=datadef)
feedback = prediction_pb2.Feedback(request=request, reward=1.0)
resp = app.SendFeedback(feedback, None)
def test_proto_seldon_metrics_endpoint(cls, client_gets_metrics):
def _match_label(line):
_data, value = line.split()
name, labels = _data.split()[0].split("{")
labels = labels[:-1]
return name, value, eval(f"dict({labels})")
def _iterate_metrics(text):
for line in text.split("\n"):
if not line or line[0] == "#":
continue
yield _match_label(line)
user_object = cls()
seldon_metrics = SeldonMetrics()
app = SeldonModelGRPC(user_object, seldon_metrics)
datadef = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=(2, 1), values=np.array([1, 2]))
)
request = prediction_pb2.SeldonMessage(data=datadef)
metrics_app = get_metrics_microservice(seldon_metrics)
metrics_client = metrics_app.test_client()
rv = metrics_client.get("/metrics")
assert rv.status_code == 200
assert rv.data.decode() == ""
resp = app.Predict(request, None)
assert (
"metrics" in json.loads(json_format.MessageToJson(resp))["meta"]
) == client_gets_metrics
rv = metrics_client.get("/metrics")
text = rv.data.decode()
timer_present = False
for name, value, labels in _iterate_metrics(text):
if name == "mytimer_bucket":
timer_present = True
if name == "mycounter_total":
assert value == "1.0"
assert labels["worker_id"] == str(os.getpid())
if name == "mygauge":
assert value == "100.0"
assert labels["worker_id"] == str(os.getpid())
if name == "customtag":
assert value == "200.0"
assert labels["mytag"] == "mytagvalue"
assert timer_present
def test_aggreate_bad_user_object():
user_object = UserObjectBad()
seldon_metrics = SeldonMetrics()
app = get_rest_microservice(user_object, seldon_metrics)
client = app.test_client()
rv = client.get('/aggregate?json={"seldonMessages":[{"data":{"ndarray":[1]}}]}')
logging.info(rv)
j = json.loads(rv.data)
logging.info(j)
assert rv.status_code == 400
assert j["status"]["info"] == "Aggregate not defined"
def test_model_ok_with_names():
user_object = UserObject()
seldon_metrics = SeldonMetrics()
app = get_rest_microservice(user_object, seldon_metrics)
client = app.test_client()
rv = client.get('/predict?json={"data":{"names":["a","b"],"ndarray":[[1,2]]}}')
j = json.loads(rv.data)
assert rv.status_code == 200
assert j["meta"]["tags"] == {"mytag": 1}
assert j["meta"]["metrics"][0]["key"] == user_object.metrics()[0]["key"]
assert j["meta"]["metrics"][0]["value"] == user_object.metrics()[0]["value"]
def test_transform_output_proto_bin_data_nparray():
user_object = UserObject(ret_nparray=True)
seldon_metrics = SeldonMetrics()
app = SeldonModelGRPC(user_object, seldon_metrics)
binData = b"\0\1"
request = prediction_pb2.SeldonMessage(binData=binData)
resp = app.TransformOutput(request, None)
jStr = json_format.MessageToJson(resp)
j = json.loads(jStr)
logging.info(j)
assert j["data"]["tensor"]["values"] == list(user_object.nparray.flatten())
def test_router_meta_to_nonmeta_model():
user_object = MinimalUserObject()
seldon_metrics = SeldonMetrics()
app = get_rest_microservice(user_object, seldon_metrics)
client = app.test_client()
rv = client.get(
'/route?json={"meta":{"puid": "abc"}, "data":{"ndarray":[2]}}')
j = json.loads(rv.data)
logging.info(j)
assert rv.status_code == 200
assert j["data"]["ndarray"] == [[22]]
def test_aggregate_bad_metrics():
user_object = UserObject(metrics_ok=False)
seldon_metrics = SeldonMetrics()
app = get_rest_microservice(user_object, seldon_metrics)
client = app.test_client()
rv = client.get(
'/aggregate?json={"seldonMessages":[{"data":{"ndarray":[1]}},{"data":{"ndarray":[2]}}]}'
)
j = json.loads(rv.data)
logging.info(j)
assert rv.status_code == 500
def test_requestPath_ok():
user_object = UserObject()
seldon_metrics = SeldonMetrics()
app = get_rest_microservice(user_object, seldon_metrics)
client = app.test_client()
rv = client.get(
'/predict?json={"meta":{"puid":"123"},"data":{"names":["a","b"],"ndarray":[[1,2]]}}'
)
j = json.loads(rv.data)
logging.info(j)
assert rv.status_code == 200
assert j["meta"]["requestPath"] == {"my-test-model": "my-test-model-image"}
def test_router_gets_meta():
user_object = UserObject(ret_meta=True)
seldon_metrics = SeldonMetrics()
app = get_rest_microservice(user_object, seldon_metrics)
client = app.test_client()
rv = client.get('/route?json={"meta":{"puid": "abc"}, "data":{"ndarray":[2]}}')
j = json.loads(rv.data)
logging.info(j)
assert rv.status_code == 200
assert j["meta"]["tags"] == {"inc_meta": {"puid": "abc"}}
assert j["meta"]["metrics"][0]["key"] == user_object.metrics()[0]["key"]
assert j["meta"]["metrics"][0]["value"] == user_object.metrics()[0]["value"]
def test_aggregate_proto_combines_metrics():
user_object = UserObject()
seldon_metrics = SeldonMetrics()
app = SeldonModelGRPC(user_object, seldon_metrics)
arr1 = np.array([1, 2])
meta1 = prediction_pb2.Meta()
json_format.ParseDict(
{
"metrics": [{
"key": "request_gauge_1",
"type": "GAUGE",
"value": 100
}]
}, meta1)
datadef1 = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr1))
arr2 = np.array([3, 4])
meta2 = prediction_pb2.Meta()
json_format.ParseDict(
{
"metrics": [{
"key": "request_gauge_2",
"type": "GAUGE",
"value": 200
}]
}, meta2)
datadef2 = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr2))
msg1 = prediction_pb2.SeldonMessage(data=datadef1, meta=meta1)
msg2 = prediction_pb2.SeldonMessage(data=datadef2, meta=meta2)
request = prediction_pb2.SeldonMessageList(seldonMessages=[msg1, msg2])
resp = app.Aggregate(request, None)
jStr = json_format.MessageToJson(resp)
j = json.loads(jStr)
logging.info(j)
assert j["meta"]["tags"] == {"mytag": 1}
assert j["meta"]["metrics"][0]["key"] == "request_gauge_1"
assert j["meta"]["metrics"][0]["value"] == 100
assert j["meta"]["metrics"][1]["key"] == "request_gauge_2"
assert j["meta"]["metrics"][1]["value"] == 200
assert j["meta"]["metrics"][2]["key"] == user_object.metrics()[0]["key"]
assert j["meta"]["metrics"][2]["value"] == user_object.metrics(
)[0]["value"]
assert j["data"]["tensor"]["shape"] == [2, 1]
assert j["data"]["tensor"]["values"] == [1, 2]
def test_seldon_metrics_endpoint(cls, client_gets_metrics):
def _match_label(line):
_data, value = line.split()
name, labels = _data.split()[0].split("{")
labels = labels[:-1]
return name, value, eval(f"dict({labels})")
def _iterate_metrics(text):
for line in text.split("\n"):
if not line or line[0] == "#":
continue
yield _match_label(line)
user_object = cls()
seldon_metrics = SeldonMetrics()
app = get_rest_microservice(user_object, seldon_metrics)
client = app.test_client()
metrics_app = get_metrics_microservice(seldon_metrics)
metrics_client = metrics_app.test_client()
rv = metrics_client.get("/metrics")
assert rv.status_code == 200
assert rv.data.decode() == ""
rv = client.get(
'/predict?json={"data": {"names": ["input"], "ndarray": ["data"]}}')
assert rv.status_code == 200
assert ("metrics" in json.loads(rv.data)["meta"]) == client_gets_metrics
rv = metrics_client.get("/metrics")
text = rv.data.decode()
timer_present = False
for name, value, labels in _iterate_metrics(text):
if name == "mytimer_bucket":
timer_present = True
if name == "mycounter_total":
assert value == "1.0"
assert labels["worker_id"] == str(os.getpid())
if name == "mygauge":
assert value == "100.0"
assert labels["worker_id"] == str(os.getpid())
if name == "customtag":
assert value == "200.0"
assert labels["mytag"] == "mytagvalue"
assert timer_present
def test_aggreate_ok_lowlevel():
user_object = UserObjectLowLevel()
seldon_metrics = SeldonMetrics()
app = get_rest_microservice(user_object, seldon_metrics)
client = app.test_client()
rv = client.get(
'/aggregate?json={"seldonMessages":[{"data":{"ndarray":[1]}},{"data":{"ndarray":[2]}}]}'
)
logging.info(rv)
j = json.loads(rv.data)
logging.info(j)
assert rv.status_code == 200
assert j["data"]["ndarray"] == [9, 9]
def test_model_str_data_identity():
user_object = UserObject()
seldon_metrics = SeldonMetrics()
app = get_rest_microservice(user_object, seldon_metrics)
client = app.test_client()
rv = client.get('/predict?json={"strData":"my data"}')
j = json.loads(rv.data)
logging.info(j)
assert rv.status_code == 200
assert j["strData"] == "my data"
assert j["meta"]["tags"] == {"mytag": 1}
assert j["meta"]["metrics"][0]["key"] == user_object.metrics()[0]["key"]
assert j["meta"]["metrics"][0]["value"] == user_object.metrics()[0]["value"]
def test_model_metadata_ok_grpc():
user_object = UserObject()
seldon_metrics = SeldonMetrics()
app = SeldonModelGRPC(user_object, seldon_metrics)
resp = app.Metadata(None, None)
assert json.loads(json_format.MessageToJson(resp)) == {
"name": "my-model-name",
"versions": ["model-version"],
"platform": "model-platform",
"inputs": [{"name": "input", "datatype": "BYTES", "shape": ["1"]}],
"outputs": [{"name": "output", "datatype": "BYTES", "shape": ["1"]}],
}
def test_model_metadata_ok():
user_object = UserObject()
seldon_metrics = SeldonMetrics()
app = get_rest_microservice(user_object, seldon_metrics)
client = app.test_client()
rv = client.get('/predict?json={"data": {"names": ["input"], "ndarray": ["data"]}}')
assert rv.status_code == 200
assert json.loads(rv.data)["data"]["ndarray"] == ["data"]
rv = client.get("/metadata")
assert rv.status_code == 200
def test_model_bin_data():
user_object = UserObject()
seldon_metrics = SeldonMetrics()
app = get_rest_microservice(user_object, seldon_metrics)
client = app.test_client()
bdata = b"123"
bdata_base64 = base64.b64encode(bdata).decode("utf-8")
rv = client.get('/predict?json={"binData":"' + bdata_base64 + '"}')
j = json.loads(rv.data)
assert rv.status_code == 200
assert j["binData"] == bdata_base64
assert j["meta"]["tags"] == {"mytag": 1}
assert j["meta"]["metrics"][0]["key"] == user_object.metrics()[0]["key"]
assert j["meta"]["metrics"][0]["value"] == user_object.metrics()[0]["value"]
def test_feedback_v01_ok():
user_object = UserObject()
seldon_metrics = SeldonMetrics()
app = get_rest_microservice(user_object, seldon_metrics)
client = app.test_client()
payload = {
"request": {"data": {"names": ["a", "b"], "ndarray": [[1, 2]]}},
"reward": 1.0,
}
rv = client.post("/api/v0.1/feedback", json=payload)
j = json.loads(rv.data)
logging.info(j)
assert rv.status_code == 200
def test_transform_output_proto_lowlevel_ok():
user_object = UserObjectLowLevelGrpc()
seldon_metrics = SeldonMetrics()
app = SeldonModelGRPC(user_object, seldon_metrics)
arr = np.array([1, 2])
datadef = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr))
request = prediction_pb2.SeldonMessage(data=datadef)
resp = app.TransformOutput(request, None)
jStr = json_format.MessageToJson(resp)
j = json.loads(jStr)
logging.info(j)
assert j["data"]["tensor"]["shape"] == [2, 1]
assert j["data"]["tensor"]["values"] == [9, 9]
def test_transformer_output_ok():
user_object = UserObject()
seldon_metrics = SeldonMetrics()
app = get_rest_microservice(user_object, seldon_metrics)
client = app.test_client()
rv = client.get('/transform-output?json={"data":{"ndarray":[1]}}')
j = json.loads(rv.data)
logging.info(j)
assert rv.status_code == 200
assert j["meta"]["tags"] == {"mytag": 1}
assert j["meta"]["metrics"][0]["key"] == user_object.metrics()[0]["key"]
assert j["meta"]["metrics"][0]["value"] == user_object.metrics(
)[0]["value"]
assert j["data"]["ndarray"] == [1]
def test_model_bin_data_nparray():
user_object = UserObject(ret_nparray=True)
seldon_metrics = SeldonMetrics()
app = get_rest_microservice(user_object, seldon_metrics)
client = app.test_client()
encoded = base64.b64encode(b"1234").decode("utf-8")
rv = client.get('/predict?json={"binData":"' + encoded + '"}')
j = json.loads(rv.data)
logging.info(j)
assert rv.status_code == 200
assert j["data"]["tensor"]["values"] == [1, 2, 3]
assert j["meta"]["tags"] == {"mytag": 1}
assert j["meta"]["metrics"][0]["key"] == user_object.metrics()[0]["key"]
assert j["meta"]["metrics"][0]["value"] == user_object.metrics()[0]["value"]
def test_model_error_status_code():
class ErrorUserObject:
def predict(self, X, features_names, **kwargs):
raise SeldonMicroserviceException("foo", status_code=403)
user_object = ErrorUserObject()
seldon_metrics = SeldonMetrics()
app = get_rest_microservice(user_object, seldon_metrics)
client = app.test_client()
uo = UserObject()
rv = client.get('/predict?json={"strData":"my data"}')
j = json.loads(rv.data)
logging.info(j)
assert rv.status_code == 403
