Exemplo n.º 1
0
def test_aggregate_proto_combines_metrics():
    user_object = UserObject()
    seldon_metrics = SeldonMetrics()
    app = SeldonModelGRPC(user_object, seldon_metrics)

    arr1 = np.array([1, 2])
    meta1 = prediction_pb2.Meta()
    json_format.ParseDict(
        {
            "metrics": [{
                "key": "request_gauge_1",
                "type": "GAUGE",
                "value": 100
            }]
        }, meta1)
    datadef1 = prediction_pb2.DefaultData(
        tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr1))

    arr2 = np.array([3, 4])
    meta2 = prediction_pb2.Meta()
    json_format.ParseDict(
        {
            "metrics": [{
                "key": "request_gauge_2",
                "type": "GAUGE",
                "value": 200
            }]
        }, meta2)
    datadef2 = prediction_pb2.DefaultData(
        tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr2))

    msg1 = prediction_pb2.SeldonMessage(data=datadef1, meta=meta1)
    msg2 = prediction_pb2.SeldonMessage(data=datadef2, meta=meta2)
    request = prediction_pb2.SeldonMessageList(seldonMessages=[msg1, msg2])
    resp = app.Aggregate(request, None)
    jStr = json_format.MessageToJson(resp)
    j = json.loads(jStr)
    logging.info(j)

    assert j["meta"]["tags"] == {"mytag": 1}

    assert j["meta"]["metrics"][0]["key"] == "request_gauge_1"
    assert j["meta"]["metrics"][0]["value"] == 100

    assert j["meta"]["metrics"][1]["key"] == "request_gauge_2"
    assert j["meta"]["metrics"][1]["value"] == 200

    assert j["meta"]["metrics"][2]["key"] == user_object.metrics()[0]["key"]
    assert j["meta"]["metrics"][2]["value"] == user_object.metrics(
    )[0]["value"]

    assert j["data"]["tensor"]["shape"] == [2, 1]
    assert j["data"]["tensor"]["values"] == [1, 2]
Exemplo n.º 2
0
    def Route(self, request, context):
        if hasattr(self.user_model, "route_grpc"):
            return self.user_model.route_grpc(request)
        else:
            datadef = request.data
            features = grpc_datadef_to_array(datadef)

            routing = np.array([[route(self.user_model, features, datadef.names)]])
            # TODO: check that predictions is 2 dimensional
            class_names = []

            data = array_to_grpc_datadef(
                routing, class_names, request.data.WhichOneof("data_oneof"))

            # Construct meta data
            meta = prediction_pb2.Meta()
            metaJson = {}
            tags = get_custom_tags(self.user_model)
            if tags:
                metaJson["tags"] = tags
            metrics = get_custom_metrics(self.user_model)
            if metrics:
                metaJson["metrics"] = metrics
            json_format.ParseDict(metaJson, meta)

            return prediction_pb2.SeldonMessage(data=data, meta=meta)
def test_transform_proto_output_passes_through_metrics():
    user_object = UserObject()
    seldon_metrics = SeldonMetrics()
    app = SeldonModelGRPC(user_object, seldon_metrics)
    arr = np.array([1, 2])
    datadef = prediction_pb2.DefaultData(
        tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr))
    meta = prediction_pb2.Meta()
    json_format.ParseDict(
        {"metrics": [{
            "key": "request_gauge",
            "type": "GAUGE",
            "value": 100
        }]}, meta)
    request = prediction_pb2.SeldonMessage(data=datadef, meta=meta)
    resp = app.TransformOutput(request, None)
    jStr = json_format.MessageToJson(resp)
    j = json.loads(jStr)
    logging.info(j)
    assert j["meta"]["metrics"][0]["key"] == "request_gauge"
    assert j["meta"]["metrics"][0]["value"] == 100

    assert j["meta"]["metrics"][1]["key"] == user_object.metrics()[0]["key"]
    assert j["meta"]["metrics"][1]["value"] == user_object.metrics(
    )[0]["value"]

    assert j["data"]["tensor"]["shape"] == [2, 1]
    assert j["data"]["tensor"]["values"] == [1, 2]
Exemplo n.º 4
0
def construct_response(user_model: SeldonComponent, is_request: bool, client_request: prediction_pb2.SeldonMessage,
                       client_raw_response: Union[np.ndarray, str, bytes, dict]) -> prediction_pb2.SeldonMessage:
    """

    Parameters
    ----------
    user_model
       Client user class
    is_request
       Whether this is part of the request flow as opposed to the response flow
    client_request
       The request received
    client_raw_response
       The raw client response from their model

    Returns
    -------
       A SeldonMessage proto response

    """
    data_type = client_request.WhichOneof("data_oneof")
    meta = prediction_pb2.Meta()
    meta_json: Dict = {}
    tags = client_custom_tags(user_model)
    if tags:
        meta_json["tags"] = tags
    metrics = client_custom_metrics(user_model)
    if metrics:
        meta_json["metrics"] = metrics
    if client_request.meta:
        if client_request.meta.puid:
            meta_json["puid"] = client_request.meta.puid
    json_format.ParseDict(meta_json, meta)
    if isinstance(client_raw_response, np.ndarray) or isinstance(client_raw_response, list):
        client_raw_response = np.array(client_raw_response)
        if is_request:
            names = client_feature_names(user_model, client_request.data.names)
        else:
            names = client_class_names(user_model, client_raw_response)
        if data_type == "data":  # If request is using defaultdata then return what was sent if is numeric response else ndarray
            if np.issubdtype(client_raw_response.dtype, np.number):
                default_data_type = client_request.data.WhichOneof("data_oneof")
            else:
                default_data_type = "ndarray"
        else:  # If numeric response return as tensor else return as ndarray
            if np.issubdtype(client_raw_response.dtype, np.number):
                default_data_type = "tensor"
            else:
                default_data_type = "ndarray"
        data = array_to_grpc_datadef(default_data_type, client_raw_response, names)
        return prediction_pb2.SeldonMessage(data=data, meta=meta)
    elif isinstance(client_raw_response, str):
        return prediction_pb2.SeldonMessage(strData=client_raw_response, meta=meta)
    elif isinstance(client_raw_response, dict):
        jsonDataResponse = ParseDict(client_raw_response, prediction_pb2.SeldonMessage().jsonData)
        return prediction_pb2.SeldonMessage(jsonData=jsonDataResponse, meta=meta)
    elif isinstance(client_raw_response, (bytes, bytearray)):
        return prediction_pb2.SeldonMessage(binData=client_raw_response, meta=meta)
    else:
        raise SeldonMicroserviceException("Unknown data type returned as payload:" + client_raw_response)
Exemplo n.º 5
0
def test_proto_tags():
    metric = {
        "tags": {
            "t1": "t2"
        },
        "metrics": [
            {
                "type": "COUNTER",
                "key": "mycounter",
                "value": 1.2
            },
            {
                "type": "GAUGE",
                "key": "mygauge",
                "value": 1.2
            },
            {
                "type": "TIMER",
                "key": "mytimer",
                "value": 1.2
            },
        ],
    }
    meta = prediction_pb2.Meta()
    json_format.ParseDict(metric, meta)
    jStr = json_format.MessageToJson(meta)
    j = json.loads(jStr)
    assert "mycounter" == j["metrics"][0]["key"]
    assert 1.2 == pytest.approx(j["metrics"][0]["value"], 0.01)
    assert "GAUGE" == j["metrics"][1]["type"]
    assert "mygauge" == j["metrics"][1]["key"]
    assert 1.2 == pytest.approx(j["metrics"][1]["value"], 0.01)
    assert "TIMER" == j["metrics"][2]["type"]
    assert "mytimer" == j["metrics"][2]["key"]
    assert 1.2 == pytest.approx(j["metrics"][2]["value"], 0.01)
Exemplo n.º 6
0
def test_model_template_app_grpc_metrics(tracing):
    with start_microservice(join(dirname(__file__), "model-template-app"),
                            tracing=tracing,
                            grpc=True):
        data = np.array([[1, 2]])
        datadef = prediction_pb2.DefaultData(tensor=prediction_pb2.Tensor(
            shape=data.shape, values=data.flatten()))

        meta = prediction_pb2.Meta()
        json_format.ParseDict(
            {"metrics": [{
                "key": "mygauge",
                "type": "GAUGE",
                "value": 100
            }]}, meta)

        request = prediction_pb2.SeldonMessage(data=datadef, meta=meta)
        channel = grpc.insecure_channel("0.0.0.0:5000")
        stub = prediction_pb2_grpc.ModelStub(channel)
        response = retry_method(stub.Predict, kwargs=dict(request=request))
        assert response.data.tensor.shape[0] == 1
        assert response.data.tensor.shape[1] == 2
        assert response.data.tensor.values[0] == 1
        assert response.data.tensor.values[1] == 2

        assert response.meta.metrics[0].key == "mygauge"
        assert response.meta.metrics[0].value == 100
Exemplo n.º 7
0
    def TransformOutput(self, request, context):
        if hasattr(self.user_model, "transform_output_grpc"):
            return self.user_model.transform_output_grpc(request)
        else:
            features = get_data_from_proto(request)
            datadef = request.data
            data_type = request.WhichOneof("data_oneof")

            # Construct meta data
            meta = prediction_pb2.Meta()
            metaJson = {}
            tags = get_custom_tags(self.user_model)
            if tags:
                metaJson["tags"] = tags
            metrics = get_custom_metrics(self.user_model)
            if metrics:
                metaJson["metrics"] = metrics
            json_format.ParseDict(metaJson, meta)

            transformed = transform_output(self.user_model, features,
                                           datadef.names)

            if isinstance(transformed, np.ndarray) or data_type == "data":
                transformed = np.array(transformed)
                class_names = get_class_names(self.user_model, datadef.names)
                if data_type == "data":
                    default_data_type = request.data.WhichOneof("data_oneof")
                else:
                    default_data_type = "tensor"
                data = array_to_grpc_datadef(transformed, class_names,
                                             default_data_type)
                return prediction_pb2.SeldonMessage(data=data, meta=meta)
            else:
                return prediction_pb2.SeldonMessage(binData=transformed,
                                                    meta=meta)
Exemplo n.º 8
0
    def route_raw(self, msg):
        logging.info("Route raw called")

        meta = prediction_pb2.Meta()
        json_format.ParseDict({"metrics": self._metrics}, meta)

        arr = np.array([22])
        datadef = prediction_pb2.DefaultData(
            tensor=prediction_pb2.Tensor(shape=(1, 1), values=arr))
        request = prediction_pb2.SeldonMessage(data=datadef, meta=meta)
        return request
def test_aggregate_proto_combines_tags():
    user_object = UserObject()
    seldon_metrics = SeldonMetrics()
    app = SeldonModelGRPC(user_object, seldon_metrics)

    arr1 = np.array([1, 2])
    meta1 = prediction_pb2.Meta()
    json_format.ParseDict({"tags": {"input-1": "yes", "common": 1}}, meta1)
    datadef1 = prediction_pb2.DefaultData(
        tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr1)
    )

    arr2 = np.array([3, 4])
    meta2 = prediction_pb2.Meta()
    json_format.ParseDict({"tags": {"input-2": "yes", "common": 2}}, meta2)
    datadef2 = prediction_pb2.DefaultData(
        tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr2)
    )

    msg1 = prediction_pb2.SeldonMessage(data=datadef1, meta=meta1)
    msg2 = prediction_pb2.SeldonMessage(data=datadef2, meta=meta2)
    request = prediction_pb2.SeldonMessageList(seldonMessages=[msg1, msg2])
    resp = app.Aggregate(request, None)
    jStr = json_format.MessageToJson(resp)
    j = json.loads(jStr)
    logging.info(j)

    assert j["meta"]["tags"] == {
        "common": 2,
        "input-1": "yes",
        "input-2": "yes",
        "mytag": 1,
    }

    # add default type
    assert j["meta"]["metrics"][0]["key"] == user_object.metrics()[0]["key"]
    assert j["meta"]["metrics"][0]["value"] == user_object.metrics()[0]["value"]
    assert j["data"]["tensor"]["shape"] == [2, 1]
    assert j["data"]["tensor"]["values"] == [1, 2]
Exemplo n.º 10
0
def test_proto_requestPath_ok():
    user_object = UserObject()
    seldon_metrics = SeldonMetrics()
    app = SeldonModelGRPC(user_object, seldon_metrics)
    arr = np.array([1, 2])
    datadef = prediction_pb2.DefaultData(
        tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr))
    meta = prediction_pb2.Meta()
    json_format.ParseDict({"tags": {"foo": "bar"}}, meta)
    request = prediction_pb2.SeldonMessage(data=datadef, meta=meta)
    resp = app.Predict(request, None)
    jStr = json_format.MessageToJson(resp)
    j = json.loads(jStr)
    logging.info(j)
    assert j["meta"]["requestPath"] == {"my-test-model": "my-test-model-image"}
def test_proto_feedback():
    user_object = UserObject()
    app = SeldonModelGRPC(user_object)
    arr = np.array([1, 2])
    datadef = prediction_pb2.DefaultData(
        tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr))
    meta = prediction_pb2.Meta()
    metaJson = {}
    routing = {"1": 1}
    metaJson["routing"] = routing
    json_format.ParseDict(metaJson, meta)

    request = prediction_pb2.SeldonMessage(data=datadef)
    response = prediction_pb2.SeldonMessage(meta=meta, data=datadef)
    feedback = prediction_pb2.Feedback(request=request,
                                       response=response,
                                       reward=1.0)
    resp = app.SendFeedback(feedback, None)
Exemplo n.º 12
0
def test_model_template_app_grpc_tags(microservice):
    data = np.array([[1, 2]])
    datadef = prediction_pb2.DefaultData(
        tensor=prediction_pb2.Tensor(shape=data.shape, values=data.flatten()))

    meta = prediction_pb2.Meta()
    json_format.ParseDict({"tags": {"foo": "bar"}}, meta)

    request = prediction_pb2.SeldonMessage(data=datadef, meta=meta)
    channel = grpc.insecure_channel("0.0.0.0:5000")
    stub = prediction_pb2_grpc.ModelStub(channel)
    response = retry_method(stub.Predict, kwargs=dict(request=request))
    assert response.data.tensor.shape[0] == 1
    assert response.data.tensor.shape[1] == 2
    assert response.data.tensor.values[0] == 1
    assert response.data.tensor.values[1] == 2

    assert response.meta.tags["foo"].string_value == "bar"
def test_proto_passes_through_tags():
    user_object = UserObject()
    app = SeldonModelGRPC(user_object)
    arr = np.array([1, 2])
    datadef = prediction_pb2.DefaultData(
        tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr))
    meta = prediction_pb2.Meta()
    json_format.ParseDict({"tags": {"foo": "bar"}}, meta)
    request = prediction_pb2.SeldonMessage(data=datadef, meta=meta)
    resp = app.Predict(request, None)
    jStr = json_format.MessageToJson(resp)
    j = json.loads(jStr)
    logging.info(j)
    assert j["meta"]["tags"] == {"foo": "bar", "mytag": 1}
    assert j["meta"]["metrics"][0]["key"] == user_object.metrics()[0]["key"]
    assert j["meta"]["metrics"][0]["value"] == user_object.metrics(
    )[0]["value"]
    assert j["data"]["tensor"]["shape"] == [2, 1]
    assert j["data"]["tensor"]["values"] == [1, 2]
Exemplo n.º 14
0
    def Predict(self, request, context):
        if hasattr(self.user_model, "predict_grpc"):
            return self.user_model.predict_grpc(request)
        else:
            features = get_data_from_proto(request)
            meta = get_meta_from_proto(request)
            datadef = request.data
            data_type = request.WhichOneof("data_oneof")
            predictions = predict(self.user_model,
                                  features,
                                  datadef.names,
                                  meta=meta)

            # Construct meta data
            meta = prediction_pb2.Meta()
            metaJson = {}
            tags = get_custom_tags(self.user_model)
            if tags:
                metaJson["tags"] = tags
            metrics = get_custom_metrics(self.user_model)
            if metrics:
                metaJson["metrics"] = metrics
            json_format.ParseDict(metaJson, meta)

            if isinstance(predictions, np.ndarray) or data_type == "data":
                predictions = np.array(predictions)
                if len(predictions.shape) > 1:
                    class_names = get_class_names(self.user_model,
                                                  predictions.shape[1])
                else:
                    class_names = []

                if data_type == "data":
                    default_data_type = request.data.WhichOneof("data_oneof")
                else:
                    default_data_type = "tensor"
                data = array_to_grpc_datadef(predictions, class_names,
                                             default_data_type)
                return prediction_pb2.SeldonMessage(data=data, meta=meta)
            else:
                return prediction_pb2.SeldonMessage(binData=predictions,
                                                    meta=meta)
Exemplo n.º 15
0
def test_proto_gets_meta():
    user_object = UserObject(ret_meta=True)
    app = SeldonModelGRPC(user_object)
    arr = np.array([1, 2])
    datadef = prediction_pb2.DefaultData(
        tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr))
    meta = prediction_pb2.Meta()
    metaJson = {"puid": "abc"}
    json_format.ParseDict(metaJson, meta)
    request = prediction_pb2.SeldonMessage(data=datadef, meta=meta)
    resp = app.Predict(request, None)
    jStr = json_format.MessageToJson(resp)
    j = json.loads(jStr)
    print(j)
    assert j["meta"]["tags"] == {"inc_meta": {"puid": "abc"}}
    assert j["meta"]["metrics"][0]["key"] == user_object.metrics()[0]["key"]
    assert j["meta"]["metrics"][0]["value"] == user_object.metrics(
    )[0]["value"]
    assert j["data"]["tensor"]["shape"] == [2, 1]
    assert j["data"]["tensor"]["values"] == [1, 2]
Exemplo n.º 16
0
    def Aggregate(self, request, context):
        if hasattr(self.user_model, "aggregate_grpc"):
            return self.user_model.aggregate_grpc(request)
        else:
            features_list = []
            names_list = []

            for msg in request.seldonMessages:
                features = get_data_from_proto(msg)
                features_list.append(features)
                names_list.append(msg.data.names)

            data_type = request.seldonMessages[0].WhichOneof("data_oneof")

            aggregated = aggregate(self.user_model, features_list, names_list)

            # Construct meta data
            meta = prediction_pb2.Meta()
            metaJson = {}
            tags = get_custom_tags(self.user_model)
            if tags:
                metaJson["tags"] = tags
            metrics = get_custom_metrics(self.user_model)
            if metrics:
                metaJson["metrics"] = metrics
            json_format.ParseDict(metaJson, meta)

            if isinstance(aggregated, np.ndarray) or data_type == "data":
                aggregated = np.array(aggregated)
                feature_names = get_feature_names(self.user_model, [])
                if data_type == "data":
                    default_data_type = request.seldonMessages[
                        0].data.WhichOneof("data_oneof")
                else:
                    default_data_type = "tensor"
                data = array_to_grpc_datadef(aggregated, feature_names,
                                             default_data_type)
                return prediction_pb2.SeldonMessage(data=data, meta=meta)
            else:
                return prediction_pb2.SeldonMessage(binData=aggregated,
                                                    meta=meta)
Exemplo n.º 17
0
def test_model_template_app_grpc_tags(tracing):
    with start_microservice(join(dirname(__file__), "model-template-app"),
                            tracing=tracing,
                            grpc=True):
        data = np.array([[1, 2]])
        datadef = prediction_pb2.DefaultData(tensor=prediction_pb2.Tensor(
            shape=data.shape, values=data.flatten()))

        meta = prediction_pb2.Meta()
        json_format.ParseDict({"tags": {"foo": "bar"}}, meta)

        request = prediction_pb2.SeldonMessage(data=datadef, meta=meta)
        channel = grpc.insecure_channel("0.0.0.0:5000")
        stub = prediction_pb2_grpc.ModelStub(channel)
        response = stub.Predict(request=request)
        assert response.data.tensor.shape[0] == 1
        assert response.data.tensor.shape[1] == 2
        assert response.data.tensor.values[0] == 1
        assert response.data.tensor.values[1] == 2

        assert response.meta.tags["foo"].string_value == "bar"
Exemplo n.º 18
0
def test_proto_metrics():
    metrics = [{"type": "COUNTER", "key": "a", "value": 1}]
    meta = prediction_pb2.Meta()
    for metric in metrics:
        mpb2 = meta.metrics.add()
        json_format.ParseDict(metric, mpb2)