def test_router_proto_ok(): user_object = UserObject() seldon_metrics = SeldonMetrics() app = SeldonModelGRPC(user_object, seldon_metrics) arr = np.array([1, 2]) datadef = prediction_pb2.DefaultData( tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr)) request = prediction_pb2.SeldonMessage(data=datadef) resp = app.Route(request, None) jStr = json_format.MessageToJson(resp) j = json.loads(jStr) logging.info(j) assert j["meta"]["tags"] == {"mytag": 1} # add default type assert j["meta"]["metrics"][0]["key"] == user_object.metrics()[0]["key"] assert j["meta"]["metrics"][0]["value"] == user_object.metrics( )[0]["value"] assert j["data"]["tensor"]["shape"] == [1, 1] assert j["data"]["tensor"]["values"] == [22]
def test_proto_feedback(): user_object = UserObject() seldon_metrics = SeldonMetrics() app = SeldonModelGRPC(user_object, seldon_metrics) arr = np.array([1, 2]) datadef = prediction_pb2.DefaultData( tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr)) meta = prediction_pb2.Meta() metaJson = {} routing = {"1": 1} metaJson["routing"] = routing json_format.ParseDict(metaJson, meta) request = prediction_pb2.SeldonMessage(data=datadef) response = prediction_pb2.SeldonMessage(meta=meta, data=datadef) feedback = prediction_pb2.Feedback(request=request, response=response, reward=1.0) resp = app.SendFeedback(feedback, None)
def test_aggregate_proto_lowlevel_ok(): user_object = UserObjectLowLevelGrpc() seldon_metrics = SeldonMetrics() app = SeldonModelGRPC(user_object, seldon_metrics) arr1 = np.array([1, 2]) datadef1 = prediction_pb2.DefaultData( tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr1)) arr2 = np.array([3, 4]) datadef2 = prediction_pb2.DefaultData( tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr2)) msg1 = prediction_pb2.SeldonMessage(data=datadef1) msg2 = prediction_pb2.SeldonMessage(data=datadef2) request = prediction_pb2.SeldonMessageList(seldonMessages=[msg1, msg2]) resp = app.Aggregate(request, None) jStr = json_format.MessageToJson(resp) j = json.loads(jStr) logging.info(j) assert j["data"]["tensor"]["shape"] == [2, 1] assert j["data"]["tensor"]["values"] == [9, 9]
def test_proto_requestPath_2nd_node_ok(): user_object = UserObject() seldon_metrics = SeldonMetrics() app = SeldonModelGRPC(user_object, seldon_metrics) arr = np.array([1, 2]) datadef = prediction_pb2.DefaultData( tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr) ) meta = prediction_pb2.Meta() json_format.ParseDict({"requestPath": {"earlier-node": "earlier-image"}}, meta) request = prediction_pb2.SeldonMessage(data=datadef, meta=meta) resp = app.Predict(request, None) jStr = json_format.MessageToJson(resp) j = json.loads(jStr) logging.info(j) assert j["meta"]["requestPath"] == { "my-test-model": "my-test-model-image", "earlier-node": "earlier-image", }
def test_proto_passes_through_tags(): user_object = UserObject() app = SeldonModelGRPC(user_object) arr = np.array([1, 2]) datadef = prediction_pb2.DefaultData(tensor=prediction_pb2.Tensor( shape=(2, 1), values=arr), ) meta = prediction_pb2.Meta() json_format.ParseDict({"tags": {"foo": "bar"}}, meta) request = prediction_pb2.SeldonMessage(data=datadef, meta=meta) resp = app.Predict(request, None) jStr = json_format.MessageToJson(resp) j = json.loads(jStr) logging.info(j) assert j["meta"]["tags"] == {"foo": "bar", "mytag": 1} assert j["meta"]["metrics"][0]["key"] == user_object.metrics()[0]["key"] assert j["meta"]["metrics"][0]["value"] == user_object.metrics( )[0]["value"] assert j["data"]["tensor"]["shape"] == [2, 1] assert j["data"]["tensor"]["values"] == [1, 2]
def test_proto_gets_meta(): user_object = UserObject(ret_meta=True) app = SeldonModelGRPC(user_object) arr = np.array([1, 2]) datadef = prediction_pb2.DefaultData( tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr)) meta = prediction_pb2.Meta() metaJson = {"puid": "abc"} json_format.ParseDict(metaJson, meta) request = prediction_pb2.SeldonMessage(data=datadef, meta=meta) resp = app.Predict(request, None) jStr = json_format.MessageToJson(resp) j = json.loads(jStr) print(j) assert j["meta"]["tags"] == {"inc_meta": {"puid": "abc"}} assert j["meta"]["metrics"][0]["key"] == user_object.metrics()[0]["key"] assert j["meta"]["metrics"][0]["value"] == user_object.metrics( )[0]["value"] assert j["data"]["tensor"]["shape"] == [2, 1] assert j["data"]["tensor"]["values"] == [1, 2]
def test_proto_seldon_metrics_route(cls, client_gets_metrics): user_object = cls() seldon_metrics = SeldonMetrics() app = SeldonModelGRPC(user_object, seldon_metrics) datadef = prediction_pb2.DefaultData( tensor=prediction_pb2.Tensor(shape=(2, 1), values=np.array([1, 2]))) request = prediction_pb2.SeldonMessage(data=datadef) resp = app.Route(request, None) assert ("metrics" in json.loads( json_format.MessageToJson(resp))["meta"]) == client_gets_metrics data = seldon_metrics.data[os.getpid()] verify_seldon_metrics(data, 1, [0.0202], ROUTER_METRIC_METHOD_TAG) resp = app.Route(request, None) assert ("metrics" in json.loads( json_format.MessageToJson(resp))["meta"]) == client_gets_metrics data = seldon_metrics.data[os.getpid()] verify_seldon_metrics(data, 2, [0.0202, 0.0202], ROUTER_METRIC_METHOD_TAG)
def test_proto_seldon_runtime_data_aggregate(cls, client_gets_metrics): user_object = cls() seldon_metrics = SeldonMetrics() app = SeldonModelGRPC(user_object, seldon_metrics) arr1 = np.array([1, 2]) datadef1 = prediction_pb2.DefaultData( tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr1) ) arr2 = np.array([3, 4]) datadef2 = prediction_pb2.DefaultData( tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr2) ) msg1 = prediction_pb2.SeldonMessage(data=datadef1) msg2 = prediction_pb2.SeldonMessage(data=datadef2) request = prediction_pb2.SeldonMessageList(seldonMessages=[msg1, msg2]) resp = app.Aggregate(request, None) j = json.loads(json_format.MessageToJson(resp)) assert j["data"] == { "names": ["t:0"], "tensor": {"shape": [2, 1], "values": [1.0, 2.0]}, } assert j["meta"]["tags"] == EXPECTED_TAGS assert ("metrics" in j["meta"]) == client_gets_metrics data = seldon_metrics.data[os.getpid()] verify_seldon_metrics(data, 1, [0.0202], AGGREGATE_METRIC_METHOD_TAG) resp = app.Aggregate(request, None) j = json.loads(json_format.MessageToJson(resp)) assert j["data"] == { "names": ["t:0"], "tensor": {"shape": [2, 1], "values": [1.0, 2.0]}, } assert j["meta"]["tags"] == EXPECTED_TAGS assert ("metrics" in j["meta"]) == client_gets_metrics data = seldon_metrics.data[os.getpid()] verify_seldon_metrics(data, 2, [0.0202, 0.0202], AGGREGATE_METRIC_METHOD_TAG)
def test_proto_seldon_metrics_aggregate(cls): user_object = cls() seldon_metrics = SeldonMetrics() app = SeldonModelGRPC(user_object, seldon_metrics) arr1 = np.array([1, 2]) datadef1 = prediction_pb2.DefaultData( tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr1)) arr2 = np.array([3, 4]) datadef2 = prediction_pb2.DefaultData( tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr2)) msg1 = prediction_pb2.SeldonMessage(data=datadef1) msg2 = prediction_pb2.SeldonMessage(data=datadef2) request = prediction_pb2.SeldonMessageList(seldonMessages=[msg1, msg2]) app.Aggregate(request, None) data = seldon_metrics.data[os.getpid()] assert data["GAUGE", "mygauge"]["value"] == 100 assert data["GAUGE", "customtag"]["value"] == 200 assert data["GAUGE", "customtag"]["tags"] == {"mytag": "mytagvalue"} assert data["COUNTER", "mycounter"]["value"] == 1 assert np.allclose( np.histogram([20.2 / 1000], BINS)[0], data["TIMER", "mytimer"]["value"][0]) assert np.allclose(data["TIMER", "mytimer"]["value"][1], 0.0202) app.Aggregate(request, None) data = seldon_metrics.data[os.getpid()] assert data["GAUGE", "mygauge"]["value"] == 100 assert data["GAUGE", "customtag"]["value"] == 200 assert data["GAUGE", "customtag"]["tags"] == {"mytag": "mytagvalue"} assert data["COUNTER", "mycounter"]["value"] == 2 assert np.allclose( np.histogram([20.2 / 1000, 20.2 / 1000], BINS)[0], data["TIMER", "mytimer"]["value"][0], ) assert np.allclose(data["TIMER", "mytimer"]["value"][1], 0.0404)
def test_proto_seldon_runtime_data_transform_output(cls, client_gets_metrics): user_object = cls() seldon_metrics = SeldonMetrics() app = SeldonModelGRPC(user_object, seldon_metrics) datadef = prediction_pb2.DefaultData( tensor=prediction_pb2.Tensor(shape=(2, 1), values=np.array([1, 2]))) request = prediction_pb2.SeldonMessage(data=datadef) resp = app.TransformOutput(request, None) j = json.loads(json_format.MessageToJson(resp)) assert j["data"] == { "names": ["t:0"], "tensor": { "shape": [2, 1], "values": [1.0, 2.0] }, } assert j["meta"]["tags"] == EXPECTED_TAGS assert ("metrics" in j["meta"]) == client_gets_metrics data = seldon_metrics.data[os.getpid()] verify_seldon_metrics(data, 1, [0.0202], OUTPUT_TRANSFORM_METRIC_METHOD_TAG) resp = app.TransformOutput(request, None) j = json.loads(json_format.MessageToJson(resp)) assert j["data"] == { "names": ["t:0"], "tensor": { "shape": [2, 1], "values": [1.0, 2.0] }, } assert j["meta"]["tags"] == EXPECTED_TAGS assert ("metrics" in j["meta"]) == client_gets_metrics data = seldon_metrics.data[os.getpid()] verify_seldon_metrics(data, 2, [0.0202, 0.0202], OUTPUT_TRANSFORM_METRIC_METHOD_TAG)
def test_aggregate_proto_combines_tags(): user_object = UserObject() seldon_metrics = SeldonMetrics() app = SeldonModelGRPC(user_object, seldon_metrics) arr1 = np.array([1, 2]) meta1 = prediction_pb2.Meta() json_format.ParseDict({"tags": {"input-1": "yes", "common": 1}}, meta1) datadef1 = prediction_pb2.DefaultData( tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr1) ) arr2 = np.array([3, 4]) meta2 = prediction_pb2.Meta() json_format.ParseDict({"tags": {"input-2": "yes", "common": 2}}, meta2) datadef2 = prediction_pb2.DefaultData( tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr2) ) msg1 = prediction_pb2.SeldonMessage(data=datadef1, meta=meta1) msg2 = prediction_pb2.SeldonMessage(data=datadef2, meta=meta2) request = prediction_pb2.SeldonMessageList(seldonMessages=[msg1, msg2]) resp = app.Aggregate(request, None) jStr = json_format.MessageToJson(resp) j = json.loads(jStr) logging.info(j) assert j["meta"]["tags"] == { "common": 2, "input-1": "yes", "input-2": "yes", "mytag": 1, } # add default type assert j["meta"]["metrics"][0]["key"] == user_object.metrics()[0]["key"] assert j["meta"]["metrics"][0]["value"] == user_object.metrics()[0]["value"] assert j["data"]["tensor"]["shape"] == [2, 1] assert j["data"]["tensor"]["values"] == [1, 2]
def test_aggregate_proto_ok(): user_object = UserObject() app = SeldonModelGRPC(user_object) arr1 = np.array([1, 2]) datadef1 = prediction_pb2.DefaultData( tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr1)) arr2 = np.array([3, 4]) datadef2 = prediction_pb2.DefaultData( tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr2)) msg1 = prediction_pb2.SeldonMessage(data=datadef1) msg2 = prediction_pb2.SeldonMessage(data=datadef2) request = prediction_pb2.SeldonMessageList(seldonMessages=[msg1, msg2]) resp = app.Aggregate(request, None) jStr = json_format.MessageToJson(resp) j = json.loads(jStr) print(j) assert j["meta"]["tags"] == {"mytag": 1} # add default type assert j["meta"]["metrics"][0]["key"] == user_object.metrics()[0]["key"] assert j["meta"]["metrics"][0]["value"] == user_object.metrics( )[0]["value"] assert j["data"]["tensor"]["shape"] == [2, 1] assert j["data"]["tensor"]["values"] == [1, 2]
def test_model_metadata_ok_grpc(): user_object = UserObject() seldon_metrics = SeldonMetrics() app = SeldonModelGRPC(user_object, seldon_metrics) resp = app.Metadata(None, None) assert json.loads(json_format.MessageToJson(resp)) == { "name": "my-model-name", "versions": ["model-version"], "platform": "model-platform", "inputs": [{ "name": "input", "datatype": "BYTES", "shape": ["1"] }], "outputs": [{ "name": "output", "datatype": "BYTES", "shape": ["1"] }], "custom": { "tag-key": "tag-value" }, }
def test_proto_seldon_metrics_endpoint(cls, client_gets_metrics): def _match_label(line): _data, value = line.split() name, labels = _data.split()[0].split("{") labels = labels[:-1] return name, value, eval(f"dict({labels})") def _iterate_metrics(text): for line in text.split("\n"): if not line or line[0] == "#": continue yield _match_label(line) user_object = cls() seldon_metrics = SeldonMetrics() app = SeldonModelGRPC(user_object, seldon_metrics) datadef = prediction_pb2.DefaultData( tensor=prediction_pb2.Tensor(shape=(2, 1), values=np.array([1, 2]))) request = prediction_pb2.SeldonMessage(data=datadef) metrics_app = get_metrics_microservice(seldon_metrics) metrics_client = metrics_app.test_client() rv = metrics_client.get("/metrics") assert rv.status_code == 200 assert rv.data.decode() == "" resp = app.Predict(request, None) j = json.loads(json_format.MessageToJson(resp)) assert j["data"] == { "names": ["t:0"], "tensor": { "shape": [2, 1], "values": [1.0, 2.0] }, } assert j["meta"]["tags"] == EXPECTED_TAGS assert ("metrics" in j["meta"]) == client_gets_metrics rv = metrics_client.get("/metrics") text = rv.data.decode() timer_present = False for name, value, labels in _iterate_metrics(text): if name == "mytimer_bucket": timer_present = True if name == "mycounter_total": assert value == "1.0" assert labels["worker_id"] == str(os.getpid()) if name == "mygauge": assert value == "100.0" assert labels["worker_id"] == str(os.getpid()) if name == "customtag": assert value == "200.0" assert labels["mytag"] == "mytagvalue" assert timer_present