def test_predict_rest_json_data_seldon(mock_post, mock_token):
    sc = SeldonClient(deployment_name="mymodel", gateway="seldon")
    response = sc.predict(json_data=JSON_TEST_DATA)
    json_response = seldon_message_to_json(response.response)
    assert "jsonData" in mock_post.call_args[1]["json"]
    assert mock_post.call_args[1]["json"]["jsonData"] == JSON_TEST_DATA
    assert response.success is True
    assert json_response["jsonData"] == JSON_TEST_DATA
    assert mock_post.call_count == 1
def test_predict_rest_with_meta(mock_post):
    sc = SeldonClient(deployment_name="mymodel")
    meta = {"key": "value"}
    response = sc.predict(names=["a", "b"], meta=meta)
    assert mock_post.call_args[1]["json"]["data"]["names"] == ["a", "b"]
    assert mock_post.call_args[1]["json"]["meta"]["tags"] == meta
    assert response.success == True
    assert response.response.data.tensor.shape == [1, 1]
    assert mock_post.call_count == 1
Esempio n. 3
0
def test_predict_rest_with_ambassador_prefix_dict_response(mock_post):
    sc = SeldonClient(deployment_name="mymodel", client_return_type="dict")
    response = sc.predict(
        gateway="ambassador", transport="rest", gateway_prefix="/mycompany/ml"
    )
    assert mock_post.call_args[0][0].index("/mycompany/ml") > 0
    assert response.success == True
    assert response.response["data"]["tensor"]["shape"] == [1, 1]
    assert mock_post.call_count == 1
 def test_tfserving(self):
     run("kubectl delete sdep --all", shell=True)
     run("kubectl apply -f ../../servers/tfserving/samples/mnist_rest.yaml", shell=True, check=True)
     wait_for_rollout("mnist-default-808f52c")
     wait_for_status("tfserving")
     print("Initial request")
     sc = SeldonClient(deployment_name="tfserving",namespace="seldon")
     r = sc.predict(gateway="ambassador",transport="rest",shape=(1,784))
     assert r.success
     print("Success for test_prepack_tfserving")
 def test_xgboost(self):
     run("kubectl delete sdep --all", shell=True)
     run("kubectl apply -f ../../servers/xgboostserver/samples/iris.yaml", shell=True, check=True)
     wait_for_rollout("iris-default-5299e79")
     wait_for_status("xgboost")
     print("Initial request")
     sc = SeldonClient(deployment_name="xgboost",namespace="seldon")
     r = sc.predict(gateway="ambassador",transport="rest",shape=(1,4))
     assert r.success
     print("Success for test_prepack_xgboost")
Esempio n. 6
0
def test_predict_rest_json_data_ambassador_dict_response(mock_post):
    sc = SeldonClient(deployment_name="mymodel",
                      gateway="ambassador",
                      client_return_type="dict")
    response = sc.predict(json_data=JSON_TEST_DATA)
    json_response = response.response
    assert "jsonData" in mock_post.call_args[1]["json"]
    assert mock_post.call_args[1]["json"]["jsonData"] == JSON_TEST_DATA
    assert response.success is True
    assert json_response["jsonData"] == JSON_TEST_DATA
    assert mock_post.call_count == 1
Esempio n. 7
0
def test_sklearn_server(data):
    y_true = [5, 0]

    sc = SeldonClient(
        gateway="ambassador",
        gateway_endpoint=API_AMBASSADOR,
        deployment_name="image-classifier",
        payload_type="ndarray",
        namespace="seldon",
        transport="rest",
    )

    sm_result = sc.predict(data=np.array(data))
    logging.info(sm_result)
    result = seldon_message_to_json(sm_result.response)
    values = result.get("data", {}).get("ndarray", {})
    assert values == labels
def test_sklearn_server():
    data = ["From: [email protected] (Brian Kantor)\nSubject: Re: HELP for Kidney Stones ..............\nOrganization: The Avant-Garde of the Now, Ltd.\nLines: 12\nNNTP-Posting-Host: ucsd.edu\n\nAs I recall from my bout with kidney stones, there isn't any\nmedication that can do anything about them except relieve the pain.\n\nEither they pass, or they have to be broken up with sound, or they have\nto be extracted surgically.\n\nWhen I was in, the X-ray tech happened to mention that she'd had kidney\nstones and children, and the childbirth hurt less.\n\nDemerol worked, although I nearly got arrested on my way home when I barfed\nall over the police car parked just outside the ER.\n\t- Brian\n",
            'From: [email protected] (David Rind)\nSubject: Re: Candida(yeast) Bloom, Fact or Fiction\nOrganization: Beth Israel Hospital, Harvard Medical School, Boston Mass., USA\nLines: 37\nNNTP-Posting-Host: enterprise.bih.harvard.edu\n\nIn article <*****@*****.**>\n [email protected] writes:\n>are in a different class.  The big question seems to be is it reasonable to \n>use them in patients with GI distress or sinus problems that *could* be due \n>to candida blooms following the use of broad-spectrum antibiotics?\n\nI guess I\'m still not clear on what the term "candida bloom" means,\nbut certainly it is well known that thrush (superficial candidal\ninfections on mucous membranes) can occur after antibiotic use.\nThis has nothing to do with systemic yeast syndrome, the "quack"\ndiagnosis that has been being discussed.\n\n\n>found in the sinus mucus membranes than is candida.  Women have been known \n>for a very long time to suffer from candida blooms in the v****a and a \n>women is lucky to find a physician who is willing to treat the cause and \n>not give give her advise to use the OTC anti-fungal creams.\n\nLucky how?  Since a recent article (randomized controlled trial) of\noral yogurt on reducing vaginal candidiasis, I\'ve mentioned to a \nnumber of patients with frequent vaginal yeast infections that they\ncould try eating 6 ounces of yogurt daily.  It turns out most would\nrather just use anti-fungal creams when they get yeast infections.\n\n>yogurt dangerous).  If this were a standard part of medical practice, as \n>Gordon R. says it is, then the incidence of GI distress and vaginal yeast \n>infections should decline.\n\nAgain, this just isn\'t what the systemic yeast syndrome is about, and\nhas nothing to do with the quack therapies that were being discussed.\nThere is some evidence that attempts to reinoculate the GI tract with\nbacteria after antibiotic therapy don\'t seem to be very helpful in\nreducing diarrhea, but I don\'t think anyone would view this as a\nquack therapy.\n-- \nDavid Rind\[email protected]\n']
    labels = [2.0, 2.0]
    
    sc = SeldonClient(
        gateway="ambassador",
        gateway_endpoint=API_AMBASSADOR,
        deployment_name="seldon-model-server",
        payload_type="ndarray",
        namespace="seldon",
        transport="rest")

    sm_result = sc.predict(data=np.array(data))
    logging.info(sm_result)
    result = seldon_message_to_json(sm_result.response)
    logging.info(result)
    values = result.get("data", {}).get("ndarray", {})
    assert (values == labels)
def test_grpc_predict_json_data_seldon(mock_get_token):
    sc = SeldonClient(deployment_name="mymodel",
                      transport="grpc",
                      gateway="seldon")
    response = sc.predict(json_data=JSON_TEST_DATA)
    assert response.response.strData == "predict"
Esempio n. 10
0
def _send_batch_predict(
    batch_idx: int,
    batch_instance_id: int,
    input_raw: str,
    data_type: str,
    sc: SeldonClient,
    retries: int,
    batch_id: str,
) -> str:
    """
    Send an request using the Seldon Client with batch context including the
    unique ID of the batch and the Batch enumerated index as metadata. This
    function also uses the unique batch ID as request ID so the request can be
    traced back individually in the Seldon Request Logger context. Each request
    will be attempted for the number of retries, and will return the string
    serialised result.
    Parameters
    ---
    batch_idx
        The enumerated index given to the batch datapoint in order of local dataset
    batch_instance_id
        The unique ID of the batch datapoint created with the python uuid function
    input_raw
        The raw input in string format to be loaded to the respective format
    data_type
        The data type to send which can be `str`, `json` and `data`
    sc
        The instance of SeldonClient to use to send the requests to the seldon model
    retries
        The number of times to retry the request
    batch_id
        The unique identifier for the batch which is passed to all requests
    Returns
    ---
        A string serialised result of the response (or equivalent data with error info)
    """

    predict_kwargs = {}
    meta = {
        "tags": {
            "batch_id": batch_id,
            "batch_instance_id": batch_instance_id,
            "batch_index": batch_idx,
        }
    }
    predict_kwargs["meta"] = meta
    predict_kwargs["headers"] = {"Seldon-Puid": batch_instance_id}
    try:
        data = json.loads(input_raw)
        if data_type == "data":
            data_np = np.array(data)
            predict_kwargs["data"] = data_np
        elif data_type == "str":
            predict_kwargs["str_data"] = data
        elif data_type == "json":
            predict_kwargs["json_data"] = data

        str_output = None
        for i in range(retries):
            try:
                seldon_payload = sc.predict(**predict_kwargs)
                assert seldon_payload.success
                str_output = json.dumps(seldon_payload.response)
                break
            except (requests.exceptions.RequestException, AssertionError) as e:
                logger.error(f"Exception: {e}, retries {retries}")
                if i == (retries - 1):
                    raise

    except Exception as e:
        error_resp = {
            "status": {
                "info": "FAILURE",
                "reason": str(e),
                "status": 1
            },
            "meta": meta,
        }
        print("Exception: %s" % e)
        str_output = json.dumps(error_resp)

    return str_output
Esempio n. 11
0
def _send_batch_predict_multi_request(
    input_data: [],
    data_type: str,
    sc: SeldonClient,
    retries: int,
    batch_id: str,
    payload_type: str,
) -> str:
    """
    Send an request using the Seldon Client with batch context including the
    unique ID of the batch and the Batch enumerated index as metadata. This
    function also uses the unique batch ID as request ID so the request can be
    traced back individually in the Seldon Request Logger context. Each request
    will be attempted for the number of retries, and will return the string
    serialised result. This method is similar to _send_batch_predict, but allows multiple
    requests to be combined into a single prediction.
    Parameters
    ---
    input_data
        The input data containing the indexes, instance_ids and predictions
    data_type
        The data type to send which can be `data`
    sc
        The instance of SeldonClient to use to send the requests to the seldon model
    retries
        The number of times to retry the request
    batch_id
        The unique identifier for the batch which is passed to all requests
    Returns
    ---
        A string serialised result of the response (or equivalent data with error info)
    """

    indexes = [x[0] for x in input_data]
    instance_ids = [x[1] for x in input_data]

    predict_kwargs = {}
    meta = {
        "tags": {
            "batch_id": batch_id,
        }
    }

    predict_kwargs["meta"] = meta
    predict_kwargs["headers"] = {"Seldon-Puid": instance_ids[0]}

    try:
        # Initialise concatenated array for data
        loaded = [json.loads(raw_data[2]) for raw_data in input_data]
        concat = np.concatenate(loaded)
        predict_kwargs["data"] = concat

        response = None
        for i in range(retries):
            try:
                seldon_payload = sc.predict(**predict_kwargs)
                assert seldon_payload.success
                response = seldon_payload.response
                break
            except (requests.exceptions.RequestException, AssertionError) as e:
                logger.error(f"Exception: {e}, retries {retries}")
                if i == (retries - 1):
                    raise

    except Exception as e:
        error_resp = {
            "status": {
                "info": "FAILURE",
                "reason": str(e),
                "status": 1
            },
            "meta": meta,
        }
        print("Exception: %s" % e)
        str_output = json.dumps(error_resp)
        return [str_output]

    # Take the response create new responses for each request
    responses = []
    # If tensor then prepare the ndarray
    tensor_ndarray = np.array(())
    if payload_type == "tensor":
        tensor = np.array(response["data"]["tensor"]["values"])
        shape = response["data"]["tensor"]["shape"]
        tensor_ndarray = tensor.reshape(shape)

    for i in range(len(input_data)):
        new_response = copy.deepcopy(response)
        if payload_type == "ndarray":
            # Format new responses for each original prediction request
            new_response["data"]["ndarray"] = [response["data"]["ndarray"][i]]
            new_response["meta"]["tags"]["tags"]["batch_index"] = indexes[i]
            new_response["meta"]["tags"]["tags"][
                "batch_instance_id"] = instance_ids[i]
            responses.append(json.dumps(new_response))
        elif payload_type == "tensor":
            # Format new responses for each original prediction request
            new_response["data"]["tensor"]["shape"][0] = 1
            new_response["data"]["tensor"]["values"] = np.ndarray.tolist(
                tensor_ndarray[i])
            new_response["meta"]["tags"]["tags"]["batch_index"] = indexes[i]
            new_response["meta"]["tags"]["tags"][
                "batch_instance_id"] = instance_ids[i]
            responses.append(json.dumps(new_response))
        else:
            raise RuntimeError(
                "Only `ndarray` and `tensor` input are currently supported for batch size greater than 1."
            )

    return responses
Esempio n. 12
0
def _send_batch_predict_multi_request(
    input_data: [],
    data_type: str,
    sc: SeldonClient,
    retries: int,
    batch_id: str,
    payload_type: str,
) -> [str]:
    """
    Send an request using the Seldon Client with batch context including the
    unique ID of the batch and the Batch enumerated index as metadata. This
    function also uses the unique batch ID as request ID so the request can be
    traced back individually in the Seldon Request Logger context. Each request
    will be attempted for the number of retries, and will return the string
    serialised result. This method is similar to _send_batch_predict, but allows multiple
    requests to be combined into a single prediction.
    Parameters
    ---
    input_data
        The input data containing the indexes, instance_ids and predictions
    data_type
        The data type to send which can be `data`
    sc
        The instance of SeldonClient to use to send the requests to the seldon model
    retries
        The number of times to retry the request
    batch_id
        The unique identifier for the batch which is passed to all requests
    Returns
    ---
        A string serialised result of the response (or equivalent data with error info)
    """

    indexes = [x[0] for x in input_data]

    seldon_puid = input_data[0][1]
    instance_ids = [f"{seldon_puid}-item-{n}" for n, _ in enumerate(input_data)]
    loaded_data = [json.loads(data[2]) for data in input_data]

    predict_kwargs = {}
    tags = {
        "batch_id": batch_id,
    }
    predict_kwargs["meta"] = tags
    predict_kwargs["headers"] = {"Seldon-Puid": seldon_puid}

    try:
        # Process raw input format
        if data_type == "raw":
            raw_data, payload_type, raw_input_tags = _extract_raw_data_multi_request(
                loaded_data, predict_kwargs["meta"]
            )
            predict_kwargs["raw_data"] = raw_data
        else:
            # Initialise concatenated array for data
            arrays = [np.array(arr) for arr in loaded_data]
            for arr in arrays:
                if arr.shape[0] != 1:
                    raise ValueError(
                        "When using mini-batching each row should contain single instance."
                    )
            concat = np.concatenate(arrays)
            predict_kwargs["data"] = concat
        logger.debug(f"calling sc.predict with {predict_kwargs}")
    except Exception as e:
        error_resp = {
            "status": {"info": "FAILURE", "reason": str(e), "status": 1},
            "meta": tags,
        }
        logger.error(f"Exception: {e}")
        str_output = json.dumps(error_resp)
        return [str_output]

    try:
        for i in range(retries):
            try:
                seldon_payload = sc.predict(**predict_kwargs)
                assert seldon_payload.success
                response = seldon_payload.response
                break
            except (requests.exceptions.RequestException, AssertionError) as e:
                logger.error(
                    f"Exception: {e}, retries {i+1} / {retries} for batch_id(s)={indexes}"
                )
                if i == (retries - 1):
                    raise

    except Exception as e:
        output = []
        for batch_index, batch_instance_id in zip(indexes, instance_ids):
            error_resp = {
                "status": {"info": "FAILURE", "reason": str(e), "status": 1},
                "meta": dict(
                    batch_index=batch_index, batch_instance_id=batch_instance_id, **tags
                ),
            }
            logger.error(f"Exception: {e}")
            output.append(json.dumps(error_resp))
        return output

    # Take the response create new responses for each request
    responses = []

    # If tensor then prepare the ndarray
    if payload_type == "tensor":
        tensor = np.array(response["data"]["tensor"]["values"])
        shape = response["data"]["tensor"]["shape"]
        tensor_ndarray = tensor.reshape(shape)

    for i in range(len(input_data)):
        try:
            new_response = copy.deepcopy(response)
            if data_type == "raw":
                new_response["meta"]["tags"].update(raw_input_tags[i])
            if payload_type == "ndarray":
                # Format new responses for each original prediction request
                new_response["data"]["ndarray"] = [response["data"]["ndarray"][i]]
                new_response["meta"]["tags"]["batch_index"] = indexes[i]
                new_response["meta"]["tags"]["batch_instance_id"] = instance_ids[i]

                responses.append(json.dumps(new_response))
            elif payload_type == "tensor":
                # Format new responses for each original prediction request
                new_response["data"]["tensor"]["shape"][0] = 1
                new_response["data"]["tensor"]["values"] = np.ndarray.tolist(
                    tensor_ndarray[i]
                )
                new_response["meta"]["tags"]["batch_index"] = indexes[i]
                new_response["meta"]["tags"]["batch_instance_id"] = instance_ids[i]
                responses.append(json.dumps(new_response))
            else:
                raise RuntimeError(
                    "Only `ndarray` and `tensor` input are currently supported for batch size greater than 1."
                )
        except Exception as e:
            error_resp = {
                "status": {"info": "FAILURE", "reason": str(e), "status": 1},
                "meta": tags,
            }
            logger.error("Exception: %s" % e)
            responses.append(json.dumps(error_resp))

    return responses
Esempio n. 13
0
def test_predict_rest(mock_post):
    sc = SeldonClient(deployment_name="mymodel")
    response = sc.predict()
    assert response.success == True
    assert response.response.data.tensor.shape == [1, 1]
    assert mock_post.call_count == 1
Esempio n. 14
0
def test_predict_grpc_seldon():
    sc = SeldonClient(deployment_name="mymodel",
                      transport="grpc",
                      gateway="seldon")
    response = sc.predict(client_return_type="proto")
    assert response.response.strData == "predict"
Esempio n. 15
0
def test_wiring_grpc_predict_ambassador(mock_grpc_predict_ambassador):
    sc = SeldonClient(deployment_name="mymodel")
    response = sc.predict(gateway="ambassador", transport="grpc")
    assert mock_grpc_predict_ambassador.call_count == 1
Esempio n. 16
0
def test_predict_grpc_ambassador_with_meta():
    sc = SeldonClient(deployment_name="mymodel",
                      transport="grpc",
                      gateway="ambassador")
    response = sc.predict(meta={"key": "value"}, client_return_type="proto")
    assert response.response.strData == "predict"
Esempio n. 17
0
from seldon_core.seldon_client import SeldonClient
import numpy as np
import matplotlib.pyplot as plt
import json
from tensorflow.examples.tutorials.mnist import input_data
import random

mnist = input_data.read_data_sets("MNIST_data/", one_hot=False)

X_train = np.vstack([img.reshape(-1,) for img in mnist.train.images])
y_train = mnist.train.labels

X_test = np.vstack([img.reshape(-1,) for img in mnist.test.images])
y_test = mnist.test.labels

sc = SeldonClient(deployment_name="mnist-model",namespace="seldon-system",gateway_endpoint="localhost:8081")
print(sc)

data = random.choice(X_test)
plt.imshow(data.reshape(28,28))
plt.colorbar()
plt.show()
r = sc.predict(data=data, gateway="istio",transport="rest")
assert(r.success==True)

res = r.response['data']['tensor']['values']
print(res)

print(int(np.argmax(np.array(res).squeeze(), axis=0)))
Esempio n. 18
0
# pip install seldon-core
from seldon_core.seldon_client import SeldonClient

path_to_image = "images/smile.jpg"
image = Image.open(path_to_image).convert("L")
resized = image.resize((64, 64))
values = np.array(resized).reshape(1, 1, 64, 64)
"""
import json
json.dump({"data": {"ndarray": values.tolist()}}, open("payload.json","w"))
"""

# this is the ip from `minikube ip` and port from `kubectl get svc ambassador -o jsonpath='{.spec.ports[0].nodePort}'`
minikube_ambassador_endpoint = "192.168.99.100:30809"

deployment_name = "seldon-emotion"
namespace = "default"

sc = SeldonClient(
    gateway="ambassador",
    gateway_endpoint=minikube_ambassador_endpoint,
    transport="rest",
    deployment_name=deployment_name,
    namespace=namespace,
)

response = sc.predict(data=values,
                      deployment_name=deployment_name,
                      payload_type="ndarray")
print(response)
Esempio n. 19
0
def test_predict_grpc_seldon(mock_get_token):
    sc = SeldonClient(deployment_name="mymodel", transport="grpc", gateway="seldon")
    response = sc.predict()
    assert response.response.strData == "predict"
    assert mock_get_token.call_count == 1
Esempio n. 20
0
def test_grpc_predict_json_data_ambassador():
    sc = SeldonClient(deployment_name="mymodel",
                      transport="grpc",
                      gateway="ambassador")
    response = sc.predict(json_data=JSON_TEST_DATA, client_return_type="proto")
    assert response.response.strData == "predict"
Esempio n. 21
0
def test_predict_grpc_ambassador():
    sc = SeldonClient(deployment_name="mymodel",
                      transport="grpc",
                      gateway="ambassador")
    response = sc.predict()
    assert response.response.strData == "predict"
Esempio n. 22
0
def test_predict_rest_404(mock_post):
    sc = SeldonClient(deployment_name="404")
    response = sc.predict()
    assert response.success == False
    assert response.msg == "404:Not Found"
Esempio n. 23
0
def test_wiring_grpc_predict_seldon_oauth(mock_grpc_predict_seldon_oauth):
    sc = SeldonClient(deployment_name="mymodel")
    response = sc.predict(gateway="seldon", transport="grpc")
    assert mock_grpc_predict_seldon_oauth.call_count == 1