Exemplo n.º 1
0
def test_proto_seldon_metrics_aggregate(cls, client_gets_metrics):
    user_object = cls()
    seldon_metrics = SeldonMetrics()

    app = SeldonModelGRPC(user_object, seldon_metrics)

    arr1 = np.array([1, 2])
    datadef1 = prediction_pb2.DefaultData(
        tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr1))
    arr2 = np.array([3, 4])
    datadef2 = prediction_pb2.DefaultData(
        tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr2))
    msg1 = prediction_pb2.SeldonMessage(data=datadef1)
    msg2 = prediction_pb2.SeldonMessage(data=datadef2)

    request = prediction_pb2.SeldonMessageList(seldonMessages=[msg1, msg2])

    resp = app.Aggregate(request, None)
    assert ("metrics" in json.loads(
        json_format.MessageToJson(resp))["meta"]) == client_gets_metrics
    data = seldon_metrics.data[os.getpid()]
    verify_seldon_metrics(data, 1, [0.0202], AGGREGATE_METRIC_METHOD_TAG)

    resp = app.Aggregate(request, None)
    assert ("metrics" in json.loads(
        json_format.MessageToJson(resp))["meta"]) == client_gets_metrics
    data = seldon_metrics.data[os.getpid()]
    verify_seldon_metrics(data, 2, [0.0202, 0.0202],
                          AGGREGATE_METRIC_METHOD_TAG)
def test_aggregate_proto_ok():
    user_object = UserObject()
    seldon_metrics = SeldonMetrics()
    app = SeldonModelGRPC(user_object, seldon_metrics)
    arr1 = np.array([1, 2])
    datadef1 = prediction_pb2.DefaultData(
        tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr1)
    )
    arr2 = np.array([3, 4])
    datadef2 = prediction_pb2.DefaultData(
        tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr2)
    )
    msg1 = prediction_pb2.SeldonMessage(data=datadef1)
    msg2 = prediction_pb2.SeldonMessage(data=datadef2)
    request = prediction_pb2.SeldonMessageList(seldonMessages=[msg1, msg2])
    resp = app.Aggregate(request, None)
    jStr = json_format.MessageToJson(resp)
    j = json.loads(jStr)
    logging.info(j)
    assert j["meta"]["tags"] == {"mytag": 1}
    # add default type
    assert j["meta"]["metrics"][0]["key"] == user_object.metrics()[0]["key"]
    assert j["meta"]["metrics"][0]["value"] == user_object.metrics()[0]["value"]
    assert j["data"]["tensor"]["shape"] == [2, 1]
    assert j["data"]["tensor"]["values"] == [1, 2]
def test_aggregate_proto_bin_data():
    user_object = UserObject()
    app = SeldonModelGRPC(user_object)
    binData = b"\0\1"
    msg1 = prediction_pb2.SeldonMessage(binData=binData)
    request = prediction_pb2.SeldonMessageList(seldonMessages=[msg1])
    resp = app.Aggregate(request, None)
    assert resp.binData == binData
Exemplo n.º 4
0
def test_aggregate_proto_combines_metrics():
    user_object = UserObject()
    seldon_metrics = SeldonMetrics()
    app = SeldonModelGRPC(user_object, seldon_metrics)

    arr1 = np.array([1, 2])
    meta1 = prediction_pb2.Meta()
    json_format.ParseDict(
        {
            "metrics": [{
                "key": "request_gauge_1",
                "type": "GAUGE",
                "value": 100
            }]
        }, meta1)
    datadef1 = prediction_pb2.DefaultData(
        tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr1))

    arr2 = np.array([3, 4])
    meta2 = prediction_pb2.Meta()
    json_format.ParseDict(
        {
            "metrics": [{
                "key": "request_gauge_2",
                "type": "GAUGE",
                "value": 200
            }]
        }, meta2)
    datadef2 = prediction_pb2.DefaultData(
        tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr2))

    msg1 = prediction_pb2.SeldonMessage(data=datadef1, meta=meta1)
    msg2 = prediction_pb2.SeldonMessage(data=datadef2, meta=meta2)
    request = prediction_pb2.SeldonMessageList(seldonMessages=[msg1, msg2])
    resp = app.Aggregate(request, None)
    jStr = json_format.MessageToJson(resp)
    j = json.loads(jStr)
    logging.info(j)

    assert j["meta"]["tags"] == {"mytag": 1}

    assert j["meta"]["metrics"][0]["key"] == "request_gauge_1"
    assert j["meta"]["metrics"][0]["value"] == 100

    assert j["meta"]["metrics"][1]["key"] == "request_gauge_2"
    assert j["meta"]["metrics"][1]["value"] == 200

    assert j["meta"]["metrics"][2]["key"] == user_object.metrics()[0]["key"]
    assert j["meta"]["metrics"][2]["value"] == user_object.metrics(
    )[0]["value"]

    assert j["data"]["tensor"]["shape"] == [2, 1]
    assert j["data"]["tensor"]["values"] == [1, 2]
Exemplo n.º 5
0
def test_proto_seldon_metrics_aggregate(cls):
    user_object = cls()
    seldon_metrics = SeldonMetrics()

    app = SeldonModelGRPC(user_object, seldon_metrics)

    arr1 = np.array([1, 2])
    datadef1 = prediction_pb2.DefaultData(
        tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr1))
    arr2 = np.array([3, 4])
    datadef2 = prediction_pb2.DefaultData(
        tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr2))
    msg1 = prediction_pb2.SeldonMessage(data=datadef1)
    msg2 = prediction_pb2.SeldonMessage(data=datadef2)

    request = prediction_pb2.SeldonMessageList(seldonMessages=[msg1, msg2])
    app.Aggregate(request, None)

    data = seldon_metrics.data[os.getpid()]
    assert data["GAUGE", "mygauge"]["value"] == 100
    assert data["GAUGE", "customtag"]["value"] == 200
    assert data["GAUGE", "customtag"]["tags"] == {"mytag": "mytagvalue"}
    assert data["COUNTER", "mycounter"]["value"] == 1
    assert np.allclose(
        np.histogram([20.2 / 1000], BINS)[0], data["TIMER",
                                                   "mytimer"]["value"][0])
    assert np.allclose(data["TIMER", "mytimer"]["value"][1], 0.0202)

    app.Aggregate(request, None)

    data = seldon_metrics.data[os.getpid()]
    assert data["GAUGE", "mygauge"]["value"] == 100
    assert data["GAUGE", "customtag"]["value"] == 200
    assert data["GAUGE", "customtag"]["tags"] == {"mytag": "mytagvalue"}
    assert data["COUNTER", "mycounter"]["value"] == 2
    assert np.allclose(
        np.histogram([20.2 / 1000, 20.2 / 1000], BINS)[0],
        data["TIMER", "mytimer"]["value"][0],
    )
    assert np.allclose(data["TIMER", "mytimer"]["value"][1], 0.0404)
def test_aggregate_proto_lowlevel_ok():
    user_object = UserObjectLowLevelGrpc()
    app = SeldonModelGRPC(user_object)
    arr1 = np.array([1, 2])
    datadef1 = prediction_pb2.DefaultData(
        tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr1))
    arr2 = np.array([3, 4])
    datadef2 = prediction_pb2.DefaultData(
        tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr2))
    msg1 = prediction_pb2.SeldonMessage(data=datadef1)
    msg2 = prediction_pb2.SeldonMessage(data=datadef2)
    request = prediction_pb2.SeldonMessageList(seldonMessages=[msg1, msg2])
    resp = app.Aggregate(request, None)
    jStr = json_format.MessageToJson(resp)
    j = json.loads(jStr)
    print(j)
    assert j["data"]["tensor"]["shape"] == [2, 1]
    assert j["data"]["tensor"]["values"] == [9, 9]
Exemplo n.º 7
0
def test_aggregate_proto_combines_tags():
    user_object = UserObject()
    app = SeldonModelGRPC(user_object)

    arr1 = np.array([1, 2])
    meta1 = prediction_pb2.Meta()
    json_format.ParseDict({"tags": {"input-1": "yes", "common": 1}}, meta1)
    datadef1 = prediction_pb2.DefaultData(
        tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr1))

    arr2 = np.array([3, 4])
    meta2 = prediction_pb2.Meta()
    json_format.ParseDict({"tags": {"input-2": "yes", "common": 2}}, meta2)
    datadef2 = prediction_pb2.DefaultData(
        tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr2))

    msg1 = prediction_pb2.SeldonMessage(data=datadef1, meta=meta1)
    msg2 = prediction_pb2.SeldonMessage(data=datadef2, meta=meta2)
    request = prediction_pb2.SeldonMessageList(seldonMessages=[msg1, msg2])
    resp = app.Aggregate(request, None)
    jStr = json_format.MessageToJson(resp)
    j = json.loads(jStr)
    logging.info(j)

    assert j["meta"]["tags"] == {
        "common": 2,
        "input-1": "yes",
        "input-2": "yes",
        "mytag": 1,
    }

    # add default type
    assert j["meta"]["metrics"][0]["key"] == user_object.metrics()[0]["key"]
    assert j["meta"]["metrics"][0]["value"] == user_object.metrics(
    )[0]["value"]
    assert j["data"]["tensor"]["shape"] == [2, 1]
    assert j["data"]["tensor"]["values"] == [1, 2]