def test_model_health_status(): user_object = UserObject() seldon_metrics = SeldonMetrics() app = get_rest_microservice(user_object, seldon_metrics) client = app.test_client() rv = client.get(HEALTH_STATUS_URL) assert rv.status_code == 200 j = json.loads(rv.data) logging.info(j) assert j["data"]["tensor"]["values"] == UserObject.HEALTH_STATUS_REPONSE
def test_transformer_output_lowlevel_raw_inherited_ok(): user_object = UserObjectLowLevelRawInherited() seldon_metrics = SeldonMetrics() app = get_rest_microservice(user_object, seldon_metrics) client = app.test_client() rv = client.get('/transform-output?json={"data":{"ndarray":[1]}}') j = json.loads(rv.data) logging.info(j) assert rv.status_code == 200 assert j["data"]["tensor"]["values"] == [9, 9]
def test_model_non200status_lowlevel(): user_object = UserObjectLowLevelWithStatusInResponse() seldon_metrics = SeldonMetrics() app = get_rest_microservice(user_object, seldon_metrics) client = app.test_client() rv = client.get( '/predict?json={"request":{"data":{"ndarray":[]}},"reward":1.0}') j = json.loads(rv.data) logging.info(j) assert rv.status_code == 400
def test_model_health_status_raw(): user_object = UserObjectLowLevel() seldon_metrics = SeldonMetrics() app = get_rest_microservice(user_object, seldon_metrics) client = app.test_client() rv = client.get(HEALTH_STATUS_URL) assert rv.status_code == 200 j = json.loads(rv.data) assert j["data"][ "ndarray"] == UserObjectLowLevel.HEALTH_STATUS_RAW_RESPONSE
def test_model_no_json(): user_object = UserObject() seldon_metrics = SeldonMetrics() app = get_rest_microservice(user_object, seldon_metrics) client = app.test_client() uo = UserObject() rv = client.get("/predict?") j = json.loads(rv.data) logging.info(j) assert rv.status_code == 400
def test_model_lowlevel_ok(): user_object = UserObjectLowLevel() seldon_metrics = SeldonMetrics() app = get_rest_microservice(user_object, seldon_metrics) client = app.test_client() rv = client.get('/predict?json={"data":{"ndarray":[1,2]}}') j = json.loads(rv.data) logging.info(j) assert rv.status_code == 200 assert j["data"]["ndarray"] == [9, 9]
def test_model_feedback_lowlevel_ok(): user_object = UserObjectLowLevel() seldon_metrics = SeldonMetrics() app = get_rest_microservice(user_object, seldon_metrics) client = app.test_client() rv = client.get( '/send-feedback?json={"request":{"data":{"ndarray":[]}},"reward":1.0}') j = json.loads(rv.data) logging.info(j) assert rv.status_code == 200
def test_model_metadata(): user_object = UserObject() seldon_metrics = SeldonMetrics() app = get_rest_microservice(user_object, seldon_metrics) client = app.test_client() rv = client.get(METADATA_URL) assert rv.status_code == 200 j = json.loads(rv.data) logging.info(j) assert j == UserObject.METADATA_RESPONSE
def test_aggreate_invalid_message(): user_object = UserObject() seldon_metrics = SeldonMetrics() app = get_rest_microservice(user_object, seldon_metrics) client = app.test_client() rv = client.get('/aggregate?json={"wrong":[{"data":{"ndarray":[1]}}]}') assert rv.status_code == 400 j = json.loads(rv.data) logging.info(j) assert j["status"]["reason"] == "MICROSERVICE_BAD_DATA"
def test_proto_feedback_custom(): user_object = UserObjectLowLevel() seldon_metrics = SeldonMetrics() app = SeldonModelGRPC(user_object, seldon_metrics) arr = np.array([1, 2]) datadef = prediction_pb2.DefaultData( tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr)) request = prediction_pb2.SeldonMessage(data=datadef) feedback = prediction_pb2.Feedback(request=request, reward=1.0) resp = app.SendFeedback(feedback, None)
def test_proto_seldon_metrics_endpoint(cls, client_gets_metrics): def _match_label(line): _data, value = line.split() name, labels = _data.split()[0].split("{") labels = labels[:-1] return name, value, eval(f"dict({labels})") def _iterate_metrics(text): for line in text.split("\n"): if not line or line[0] == "#": continue yield _match_label(line) user_object = cls() seldon_metrics = SeldonMetrics() app = SeldonModelGRPC(user_object, seldon_metrics) datadef = prediction_pb2.DefaultData( tensor=prediction_pb2.Tensor(shape=(2, 1), values=np.array([1, 2])) ) request = prediction_pb2.SeldonMessage(data=datadef) metrics_app = get_metrics_microservice(seldon_metrics) metrics_client = metrics_app.test_client() rv = metrics_client.get("/metrics") assert rv.status_code == 200 assert rv.data.decode() == "" resp = app.Predict(request, None) assert ( "metrics" in json.loads(json_format.MessageToJson(resp))["meta"] ) == client_gets_metrics rv = metrics_client.get("/metrics") text = rv.data.decode() timer_present = False for name, value, labels in _iterate_metrics(text): if name == "mytimer_bucket": timer_present = True if name == "mycounter_total": assert value == "1.0" assert labels["worker_id"] == str(os.getpid()) if name == "mygauge": assert value == "100.0" assert labels["worker_id"] == str(os.getpid()) if name == "customtag": assert value == "200.0" assert labels["mytag"] == "mytagvalue" assert timer_present
def test_aggreate_bad_user_object(): user_object = UserObjectBad() seldon_metrics = SeldonMetrics() app = get_rest_microservice(user_object, seldon_metrics) client = app.test_client() rv = client.get('/aggregate?json={"seldonMessages":[{"data":{"ndarray":[1]}}]}') logging.info(rv) j = json.loads(rv.data) logging.info(j) assert rv.status_code == 400 assert j["status"]["info"] == "Aggregate not defined"
def test_model_ok_with_names(): user_object = UserObject() seldon_metrics = SeldonMetrics() app = get_rest_microservice(user_object, seldon_metrics) client = app.test_client() rv = client.get('/predict?json={"data":{"names":["a","b"],"ndarray":[[1,2]]}}') j = json.loads(rv.data) assert rv.status_code == 200 assert j["meta"]["tags"] == {"mytag": 1} assert j["meta"]["metrics"][0]["key"] == user_object.metrics()[0]["key"] assert j["meta"]["metrics"][0]["value"] == user_object.metrics()[0]["value"]
def test_transform_output_proto_bin_data_nparray(): user_object = UserObject(ret_nparray=True) seldon_metrics = SeldonMetrics() app = SeldonModelGRPC(user_object, seldon_metrics) binData = b"\0\1" request = prediction_pb2.SeldonMessage(binData=binData) resp = app.TransformOutput(request, None) jStr = json_format.MessageToJson(resp) j = json.loads(jStr) logging.info(j) assert j["data"]["tensor"]["values"] == list(user_object.nparray.flatten())
def test_router_meta_to_nonmeta_model(): user_object = MinimalUserObject() seldon_metrics = SeldonMetrics() app = get_rest_microservice(user_object, seldon_metrics) client = app.test_client() rv = client.get( '/route?json={"meta":{"puid": "abc"}, "data":{"ndarray":[2]}}') j = json.loads(rv.data) logging.info(j) assert rv.status_code == 200 assert j["data"]["ndarray"] == [[22]]
def test_aggregate_bad_metrics(): user_object = UserObject(metrics_ok=False) seldon_metrics = SeldonMetrics() app = get_rest_microservice(user_object, seldon_metrics) client = app.test_client() rv = client.get( '/aggregate?json={"seldonMessages":[{"data":{"ndarray":[1]}},{"data":{"ndarray":[2]}}]}' ) j = json.loads(rv.data) logging.info(j) assert rv.status_code == 500
def test_requestPath_ok(): user_object = UserObject() seldon_metrics = SeldonMetrics() app = get_rest_microservice(user_object, seldon_metrics) client = app.test_client() rv = client.get( '/predict?json={"meta":{"puid":"123"},"data":{"names":["a","b"],"ndarray":[[1,2]]}}' ) j = json.loads(rv.data) logging.info(j) assert rv.status_code == 200 assert j["meta"]["requestPath"] == {"my-test-model": "my-test-model-image"}
def test_router_gets_meta(): user_object = UserObject(ret_meta=True) seldon_metrics = SeldonMetrics() app = get_rest_microservice(user_object, seldon_metrics) client = app.test_client() rv = client.get('/route?json={"meta":{"puid": "abc"}, "data":{"ndarray":[2]}}') j = json.loads(rv.data) logging.info(j) assert rv.status_code == 200 assert j["meta"]["tags"] == {"inc_meta": {"puid": "abc"}} assert j["meta"]["metrics"][0]["key"] == user_object.metrics()[0]["key"] assert j["meta"]["metrics"][0]["value"] == user_object.metrics()[0]["value"]
def test_aggregate_proto_combines_metrics(): user_object = UserObject() seldon_metrics = SeldonMetrics() app = SeldonModelGRPC(user_object, seldon_metrics) arr1 = np.array([1, 2]) meta1 = prediction_pb2.Meta() json_format.ParseDict( { "metrics": [{ "key": "request_gauge_1", "type": "GAUGE", "value": 100 }] }, meta1) datadef1 = prediction_pb2.DefaultData( tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr1)) arr2 = np.array([3, 4]) meta2 = prediction_pb2.Meta() json_format.ParseDict( { "metrics": [{ "key": "request_gauge_2", "type": "GAUGE", "value": 200 }] }, meta2) datadef2 = prediction_pb2.DefaultData( tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr2)) msg1 = prediction_pb2.SeldonMessage(data=datadef1, meta=meta1) msg2 = prediction_pb2.SeldonMessage(data=datadef2, meta=meta2) request = prediction_pb2.SeldonMessageList(seldonMessages=[msg1, msg2]) resp = app.Aggregate(request, None) jStr = json_format.MessageToJson(resp) j = json.loads(jStr) logging.info(j) assert j["meta"]["tags"] == {"mytag": 1} assert j["meta"]["metrics"][0]["key"] == "request_gauge_1" assert j["meta"]["metrics"][0]["value"] == 100 assert j["meta"]["metrics"][1]["key"] == "request_gauge_2" assert j["meta"]["metrics"][1]["value"] == 200 assert j["meta"]["metrics"][2]["key"] == user_object.metrics()[0]["key"] assert j["meta"]["metrics"][2]["value"] == user_object.metrics( )[0]["value"] assert j["data"]["tensor"]["shape"] == [2, 1] assert j["data"]["tensor"]["values"] == [1, 2]
def test_seldon_metrics_endpoint(cls, client_gets_metrics): def _match_label(line): _data, value = line.split() name, labels = _data.split()[0].split("{") labels = labels[:-1] return name, value, eval(f"dict({labels})") def _iterate_metrics(text): for line in text.split("\n"): if not line or line[0] == "#": continue yield _match_label(line) user_object = cls() seldon_metrics = SeldonMetrics() app = get_rest_microservice(user_object, seldon_metrics) client = app.test_client() metrics_app = get_metrics_microservice(seldon_metrics) metrics_client = metrics_app.test_client() rv = metrics_client.get("/metrics") assert rv.status_code == 200 assert rv.data.decode() == "" rv = client.get( '/predict?json={"data": {"names": ["input"], "ndarray": ["data"]}}') assert rv.status_code == 200 assert ("metrics" in json.loads(rv.data)["meta"]) == client_gets_metrics rv = metrics_client.get("/metrics") text = rv.data.decode() timer_present = False for name, value, labels in _iterate_metrics(text): if name == "mytimer_bucket": timer_present = True if name == "mycounter_total": assert value == "1.0" assert labels["worker_id"] == str(os.getpid()) if name == "mygauge": assert value == "100.0" assert labels["worker_id"] == str(os.getpid()) if name == "customtag": assert value == "200.0" assert labels["mytag"] == "mytagvalue" assert timer_present
def test_aggreate_ok_lowlevel(): user_object = UserObjectLowLevel() seldon_metrics = SeldonMetrics() app = get_rest_microservice(user_object, seldon_metrics) client = app.test_client() rv = client.get( '/aggregate?json={"seldonMessages":[{"data":{"ndarray":[1]}},{"data":{"ndarray":[2]}}]}' ) logging.info(rv) j = json.loads(rv.data) logging.info(j) assert rv.status_code == 200 assert j["data"]["ndarray"] == [9, 9]
def test_model_str_data_identity(): user_object = UserObject() seldon_metrics = SeldonMetrics() app = get_rest_microservice(user_object, seldon_metrics) client = app.test_client() rv = client.get('/predict?json={"strData":"my data"}') j = json.loads(rv.data) logging.info(j) assert rv.status_code == 200 assert j["strData"] == "my data" assert j["meta"]["tags"] == {"mytag": 1} assert j["meta"]["metrics"][0]["key"] == user_object.metrics()[0]["key"] assert j["meta"]["metrics"][0]["value"] == user_object.metrics()[0]["value"]
def test_model_metadata_ok_grpc(): user_object = UserObject() seldon_metrics = SeldonMetrics() app = SeldonModelGRPC(user_object, seldon_metrics) resp = app.Metadata(None, None) assert json.loads(json_format.MessageToJson(resp)) == { "name": "my-model-name", "versions": ["model-version"], "platform": "model-platform", "inputs": [{"name": "input", "datatype": "BYTES", "shape": ["1"]}], "outputs": [{"name": "output", "datatype": "BYTES", "shape": ["1"]}], }
def test_model_metadata_ok(): user_object = UserObject() seldon_metrics = SeldonMetrics() app = get_rest_microservice(user_object, seldon_metrics) client = app.test_client() rv = client.get('/predict?json={"data": {"names": ["input"], "ndarray": ["data"]}}') assert rv.status_code == 200 assert json.loads(rv.data)["data"]["ndarray"] == ["data"] rv = client.get("/metadata") assert rv.status_code == 200
def test_model_bin_data(): user_object = UserObject() seldon_metrics = SeldonMetrics() app = get_rest_microservice(user_object, seldon_metrics) client = app.test_client() bdata = b"123" bdata_base64 = base64.b64encode(bdata).decode("utf-8") rv = client.get('/predict?json={"binData":"' + bdata_base64 + '"}') j = json.loads(rv.data) assert rv.status_code == 200 assert j["binData"] == bdata_base64 assert j["meta"]["tags"] == {"mytag": 1} assert j["meta"]["metrics"][0]["key"] == user_object.metrics()[0]["key"] assert j["meta"]["metrics"][0]["value"] == user_object.metrics()[0]["value"]
def test_feedback_v01_ok(): user_object = UserObject() seldon_metrics = SeldonMetrics() app = get_rest_microservice(user_object, seldon_metrics) client = app.test_client() payload = { "request": {"data": {"names": ["a", "b"], "ndarray": [[1, 2]]}}, "reward": 1.0, } rv = client.post("/api/v0.1/feedback", json=payload) j = json.loads(rv.data) logging.info(j) assert rv.status_code == 200
def test_transform_output_proto_lowlevel_ok(): user_object = UserObjectLowLevelGrpc() seldon_metrics = SeldonMetrics() app = SeldonModelGRPC(user_object, seldon_metrics) arr = np.array([1, 2]) datadef = prediction_pb2.DefaultData( tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr)) request = prediction_pb2.SeldonMessage(data=datadef) resp = app.TransformOutput(request, None) jStr = json_format.MessageToJson(resp) j = json.loads(jStr) logging.info(j) assert j["data"]["tensor"]["shape"] == [2, 1] assert j["data"]["tensor"]["values"] == [9, 9]
def test_transformer_output_ok(): user_object = UserObject() seldon_metrics = SeldonMetrics() app = get_rest_microservice(user_object, seldon_metrics) client = app.test_client() rv = client.get('/transform-output?json={"data":{"ndarray":[1]}}') j = json.loads(rv.data) logging.info(j) assert rv.status_code == 200 assert j["meta"]["tags"] == {"mytag": 1} assert j["meta"]["metrics"][0]["key"] == user_object.metrics()[0]["key"] assert j["meta"]["metrics"][0]["value"] == user_object.metrics( )[0]["value"] assert j["data"]["ndarray"] == [1]
def test_model_bin_data_nparray(): user_object = UserObject(ret_nparray=True) seldon_metrics = SeldonMetrics() app = get_rest_microservice(user_object, seldon_metrics) client = app.test_client() encoded = base64.b64encode(b"1234").decode("utf-8") rv = client.get('/predict?json={"binData":"' + encoded + '"}') j = json.loads(rv.data) logging.info(j) assert rv.status_code == 200 assert j["data"]["tensor"]["values"] == [1, 2, 3] assert j["meta"]["tags"] == {"mytag": 1} assert j["meta"]["metrics"][0]["key"] == user_object.metrics()[0]["key"] assert j["meta"]["metrics"][0]["value"] == user_object.metrics()[0]["value"]
def test_model_error_status_code(): class ErrorUserObject: def predict(self, X, features_names, **kwargs): raise SeldonMicroserviceException("foo", status_code=403) user_object = ErrorUserObject() seldon_metrics = SeldonMetrics() app = get_rest_microservice(user_object, seldon_metrics) client = app.test_client() uo = UserObject() rv = client.get('/predict?json={"strData":"my data"}') j = json.loads(rv.data) logging.info(j) assert rv.status_code == 403