def get_seldon_client(args) -> SeldonClient:
"""
Get the appropriate Seldon Client based on args
Parameters
----------
args
Command line args
Returns
-------
A Seldon Client
"""
endpoint = args.host + ":" + str(args.port)
if args.oauth_key:
if args.grpc:
seldon_grpc_endpoint = endpoint
seldon_rest_endpoint = args.host + ":" + str(args.oauth_port)
else:
seldon_grpc_endpoint = None
seldon_rest_endpoint = endpoint
sc = SeldonClient(gateway="seldon",
seldon_rest_endpoint=seldon_rest_endpoint,
seldon_grpc_endpoint=seldon_grpc_endpoint,
oauth_key=args.oauth_key,
oauth_secret=args.oauth_secret)
else:
ambassador_endpoint = endpoint
sc = SeldonClient(gateway="ambassador",
ambassador_endpoint=ambassador_endpoint,
deployment_name=args.deployment,
namespace=args.namespace)
return sc
def test_predict_rest(mock_post):
sc = SeldonClient(deployment_name="mymodel")
response = sc.predict(client_return_type="proto")
logging.info(mock_post.call_args)
assert response.success == True
assert response.response.data.tensor.shape == [1, 1]
assert mock_post.call_count == 1
def test_predict_rest_with_names(mock_post):
sc = SeldonClient(deployment_name="mymodel")
response = sc.predict(names=["a", "b"], client_return_type="proto")
assert mock_post.call_args[1]["json"]["data"]["names"] == ["a", "b"]
assert response.success == True
assert response.response.data.tensor.shape == [1, 1]
assert mock_post.call_count == 1
def test_predict_grpc_raw_data_gateway():
sc = SeldonClient(deployment_name="mymodel",
transport="grpc",
gateway="istio")
response = sc.predict(raw_data=RAW_DATA_TEST, client_return_type="proto")
request = seldon_message_to_json(response.request)
assert request == RAW_DATA_TEST
def test_predict_grpc_seldon(mock_get_token):
sc = SeldonClient(deployment_name="mymodel",
transport="grpc",
gateway="seldon")
response = sc.predict(client_return_type="proto")
assert response.response.strData == "predict"
assert mock_get_token.call_count == 1
def test():
ds = tfds.load(name="mnist", split="test", as_supervised=True)
sc = SeldonClient(deployment_name="mnist-model",
namespace="seldon-system",
gateway_endpoint="localhost:8081",
gateway="istio")
print(sc.config)
test_size = 100
corrects = 0
data = ds.take(test_size).cache()
for image, label in data:
# plt.imshow(data.reshape(28,28))
# plt.colorbar()
# plt.show()
r = sc.predict(data=np.array(image), gateway="istio", transport="rest")
print(r.msg)
assert (r.success == True)
res = r.response['data']['tensor']['values']
print(res)
prediction = int(np.argmax(np.array(res).squeeze(), axis=0))
print("predicted: ", prediction, "Truth: ", int(label))
if prediction == int(label):
corrects = corrects + 1
print(corrects / test_size)
assert (corrects / test_size > 0.9)
def run_predict(args):
"""
Make a predict call to microservice
Parameters
----------
args
Command line args
"""
contract = json.load(open(args.contract, 'r'))
contract = unfold_contract(contract)
endpoint = f"{args.host}:{args.port}"
sc = SeldonClient(microservice_endpoint=endpoint)
for i in range(args.n_requests):
batch: ndarray = generate_batch(contract, args.batch_size, 'features')
if args.prnt:
print(f"{'-' * 40}\nSENDING NEW REQUEST:\n")
print(batch)
transport = "grpc" if args.grpc else "rest"
payload_type = "tensor" if args.tensor else "ndarray"
response = sc.microservice(data=batch,
transport=transport,
method="predict",
payload_type=payload_type)
if args.prnt:
print(f"RECEIVED RESPONSE:\n{response.response}\n")
def get_seldon_client(args) -> SeldonClient:
"""
Get the appropriate Seldon Client based on args
Parameters
----------
args
Command line args
Returns
-------
A Seldon Client
"""
endpoint = args.host + ":" + str(args.port)
gateway_endpoint = endpoint
if args.grpc:
transport = "grpc"
else:
transport = "rest"
return SeldonClient(
gateway="ambassador",
gateway_endpoint=gateway_endpoint,
transport=transport,
deployment_name=args.deployment,
namespace=args.namespace,
)
def test_predict_microservice_rest(mock_post):
sc = SeldonClient(deployment_name="mymodel")
response = sc.microservice(method="predict")
logging.info(response)
assert response.success == True
assert response.response.data.tensor.shape == [1, 1]
assert mock_post.call_count == 1
def test_grpc_predict_custom_data_ambassador():
sc = SeldonClient(deployment_name="mymodel",
transport="grpc",
gateway="ambassador")
response = sc.predict(custom_data=CUSTOM_TEST_DATA,
client_return_type="proto")
assert response.response.strData == "predict"
def test_wiring_microservice_api_grpc_feedback(mock_handler):
sc = SeldonClient()
response = sc.microservice_feedback(prediction_pb2.SeldonMessage(),
prediction_pb2.SeldonMessage(),
1.0,
transport="grpc")
assert mock_handler.call_count == 1
def test_predict_rest_json_data_ambassador(mock_post):
sc = SeldonClient(deployment_name="mymodel", gateway="ambassador")
response = sc.predict(json_data=JSON_TEST_DATA, client_return_type="proto")
json_response = seldon_message_to_json(response.response)
assert "jsonData" in mock_post.call_args[1]["json"]
assert mock_post.call_args[1]["json"]["jsonData"] == JSON_TEST_DATA
assert response.success is True
assert json_response["jsonData"] == JSON_TEST_DATA
assert mock_post.call_count == 1
def test_predict_rest_raw_data_gateway_dict(mock_post):
sc = SeldonClient(deployment_name="mymodel",
gateway="istio",
client_return_type="dict")
response = sc.predict(raw_data=RAW_DATA_TEST)
json_response = response.response
assert mock_post.call_args[1]["json"] == RAW_DATA_TEST
assert response.success is True
assert mock_post.call_count == 1
def test_predict_rest_with_meta(mock_post):
sc = SeldonClient(deployment_name="mymodel")
meta = {"key": "value"}
response = sc.predict(names=["a", "b"], meta=meta)
assert mock_post.call_args[1]["json"]["data"]["names"] == ["a", "b"]
assert mock_post.call_args[1]["json"]["meta"]["tags"] == meta
assert response.success == True
assert response.response.data.tensor.shape == [1, 1]
assert mock_post.call_count == 1
def test_grpc_predict_json_data_seldon():
sc = SeldonClient(
deployment_name="mymodel",
transport="grpc",
gateway="seldon",
client_return_type="proto",
)
response = sc.predict(json_data=JSON_TEST_DATA)
assert response.response.strData == "predict"
def test_predict_rest_with_ambassador_prefix_dict_response(mock_post):
sc = SeldonClient(deployment_name="mymodel", client_return_type="dict")
response = sc.predict(gateway="ambassador",
transport="rest",
gateway_prefix="/mycompany/ml")
assert mock_post.call_args[0][0].index("/mycompany/ml") > 0
assert response.success == True
assert response.response["data"]["tensor"]["shape"] == [1, 1]
assert mock_post.call_count == 1
def test_explain_rest_json_data_ambassador(mock_post):
sc = SeldonClient(deployment_name="mymodel", gateway="ambassador")
json_response = sc.explain(json_data=JSON_TEST_DATA)
# Currently this doesn't need to convert to JSON due to #1083
# i.e. json_response = seldon_message_to_json(response.response)
assert "jsonData" in mock_post.call_args[1]["json"]
assert mock_post.call_args[1]["json"]["jsonData"] == JSON_TEST_DATA
assert json_response["jsonData"] == JSON_TEST_DATA
assert mock_post.call_count == 1
def test_predict_rest_json_data_seldon(mock_post, mock_token):
sc = SeldonClient(deployment_name="mymodel", gateway="seldon")
response = sc.predict(json_data=JSON_TEST_DATA)
json_response = seldon_message_to_json(response.response)
assert "jsonData" in mock_post.call_args[1]["json"]
assert mock_post.call_args[1]["json"]["jsonData"] == JSON_TEST_DATA
assert response.success is True
assert json_response["jsonData"] == JSON_TEST_DATA
assert mock_post.call_count == 1
def test_predict_rest_raw_data_seldon_proto(mock_post):
sc = SeldonClient(deployment_name="mymodel",
gateway="seldon",
client_return_type="proto")
response = sc.predict(raw_data=RAW_DATA_TEST)
json_response = seldon_message_to_json(response.response)
assert mock_post.call_args[1]["json"] == RAW_DATA_TEST
assert response.success is True
assert mock_post.call_count == 1
def test_grpc_predict_custom_data_seldon(mock_get_token):
sc = SeldonClient(
deployment_name="mymodel",
transport="grpc",
gateway="seldon",
client_return_type="proto",
)
response = sc.predict(custom_data=CUSTOM_TEST_DATA)
assert response.response.strData == "predict"
def test_xgboost(self):
run("kubectl delete sdep --all", shell=True)
run("kubectl apply -f ../../servers/xgboostserver/samples/iris.yaml", shell=True, check=True)
wait_for_rollout("iris-default-5299e79")
wait_for_status("xgboost")
print("Initial request")
sc = SeldonClient(deployment_name="xgboost",namespace="seldon")
r = sc.predict(gateway="ambassador",transport="rest",shape=(1,4))
assert r.success
print("Success for test_prepack_xgboost")
def test_feedback_microservice_rest(mock_post):
sc = SeldonClient(deployment_name="mymodel")
response = sc.microservice_feedback(
prediction_request=prediction_pb2.SeldonMessage(),
prediction_response=prediction_pb2.SeldonMessage(),
reward=1.0,
)
assert response.success == True
assert response.response.data.tensor.shape == [1, 1]
assert mock_post.call_count == 1
def test_predict_microservice_rest_json_data(mock_post):
sc = SeldonClient(deployment_name="mymodel")
response = sc.microservice(method="predict", json_data=JSON_TEST_DATA)
json_response = seldon_message_to_json(response.response)
assert "jsonData" in mock_post.call_args[1]["data"]["json"]
assert response.success is True
assert mock_post.call_args[1]["data"]["json"] == json.dumps(
{"jsonData": JSON_TEST_DATA})
assert json_response["jsonData"] == JSON_TEST_DATA
assert mock_post.call_count == 1
def test_tfserving(self):
run("kubectl delete sdep --all", shell=True)
run("kubectl apply -f ../../servers/tfserving/samples/mnist_rest.yaml", shell=True, check=True)
wait_for_rollout("mnist-default-808f52c")
wait_for_status("tfserving")
print("Initial request")
sc = SeldonClient(deployment_name="tfserving",namespace="seldon")
r = sc.predict(gateway="ambassador",transport="rest",shape=(1,784))
assert r.success
print("Success for test_prepack_tfserving")
def test_predict_rest_json_data_seldon_return_type(mock_post):
sc = SeldonClient(deployment_name="mymodel",
gateway="seldon",
client_return_type="dict")
response = sc.predict(json_data=JSON_TEST_DATA)
json_response = response.response
assert "jsonData" in mock_post.call_args[1]["json"]
assert mock_post.call_args[1]["json"]["jsonData"] == JSON_TEST_DATA
assert response.success is True
assert json_response["jsonData"] == JSON_TEST_DATA
assert mock_post.call_count == 1
def test_sklearn_server(data):
y_true = [5, 0]
sc = SeldonClient(
gateway="ambassador",
gateway_endpoint=API_AMBASSADOR,
deployment_name="image-classifier",
payload_type="ndarray",
namespace="seldon",
transport="rest",
)
sm_result = sc.predict(data=np.array(data))
logging.info(sm_result)
result = seldon_message_to_json(sm_result.response)
values = result.get("data", {}).get("ndarray", {})
assert values == labels
def test_sklearn_server():
data = ["From: [email protected] (Brian Kantor)\nSubject: Re: HELP for Kidney Stones ..............\nOrganization: The Avant-Garde of the Now, Ltd.\nLines: 12\nNNTP-Posting-Host: ucsd.edu\n\nAs I recall from my bout with kidney stones, there isn't any\nmedication that can do anything about them except relieve the pain.\n\nEither they pass, or they have to be broken up with sound, or they have\nto be extracted surgically.\n\nWhen I was in, the X-ray tech happened to mention that she'd had kidney\nstones and children, and the childbirth hurt less.\n\nDemerol worked, although I nearly got arrested on my way home when I barfed\nall over the police car parked just outside the ER.\n\t- Brian\n",
'From: [email protected] (David Rind)\nSubject: Re: Candida(yeast) Bloom, Fact or Fiction\nOrganization: Beth Israel Hospital, Harvard Medical School, Boston Mass., USA\nLines: 37\nNNTP-Posting-Host: enterprise.bih.harvard.edu\n\nIn article <*****@*****.**>\n [email protected] writes:\n>are in a different class. The big question seems to be is it reasonable to \n>use them in patients with GI distress or sinus problems that *could* be due \n>to candida blooms following the use of broad-spectrum antibiotics?\n\nI guess I\'m still not clear on what the term "candida bloom" means,\nbut certainly it is well known that thrush (superficial candidal\ninfections on mucous membranes) can occur after antibiotic use.\nThis has nothing to do with systemic yeast syndrome, the "quack"\ndiagnosis that has been being discussed.\n\n\n>found in the sinus mucus membranes than is candida. Women have been known \n>for a very long time to suffer from candida blooms in the v****a and a \n>women is lucky to find a physician who is willing to treat the cause and \n>not give give her advise to use the OTC anti-fungal creams.\n\nLucky how? Since a recent article (randomized controlled trial) of\noral yogurt on reducing vaginal candidiasis, I\'ve mentioned to a \nnumber of patients with frequent vaginal yeast infections that they\ncould try eating 6 ounces of yogurt daily. It turns out most would\nrather just use anti-fungal creams when they get yeast infections.\n\n>yogurt dangerous). If this were a standard part of medical practice, as \n>Gordon R. says it is, then the incidence of GI distress and vaginal yeast \n>infections should decline.\n\nAgain, this just isn\'t what the systemic yeast syndrome is about, and\nhas nothing to do with the quack therapies that were being discussed.\nThere is some evidence that attempts to reinoculate the GI tract with\nbacteria after antibiotic therapy don\'t seem to be very helpful in\nreducing diarrhea, but I don\'t think anyone would view this as a\nquack therapy.\n-- \nDavid Rind\[email protected]\n']
labels = [2.0, 2.0]
sc = SeldonClient(
gateway="ambassador",
gateway_endpoint=API_AMBASSADOR,
deployment_name="seldon-model-server",
payload_type="ndarray",
namespace="seldon",
transport="rest")
sm_result = sc.predict(data=np.array(data))
logging.info(sm_result)
result = seldon_message_to_json(sm_result.response)
logging.info(result)
values = result.get("data", {}).get("ndarray", {})
assert (values == labels)
def run_send_feedback(args):
"""
Make a feedback call to microservice
Parameters
----------
args
Command line args
"""
contract = json.load(open(args.contract, "r"))
contract = unfold_contract(contract)
endpoint = args.host + ":" + str(args.port)
sc = SeldonClient(microservice_endpoint=endpoint)
for i in range(args.n_requests):
batch = generate_batch(contract, args.batch_size, "features")
if args.prnt:
print("-" * 40)
print("SENDING NEW REQUEST:")
if not args.grpc:
transport = "rest"
else:
transport = "grpc"
if args.tensor:
payload_type = "tensor"
else:
payload_type = "ndarray"
response_predict = sc.microservice(
data=batch, transport=transport, payload_type=payload_type, method="predict"
)
response_feedback = sc.microservice_feedback(
prediction_request=response_predict.request,
prediction_response=response_predict.response,
reward=1.0,
transport=transport,
)
if args.prnt:
print(f"RECEIVED RESPONSE:\n{response_feedback}\n")
def test_predict_rest_404(mock_post):
sc = SeldonClient(deployment_name="404")
response = sc.predict()
assert response.success == False
assert response.msg == "404:Not Found"
def test_wiring_grpc_predict_ambassador(mock_grpc_predict_ambassador):
sc = SeldonClient(deployment_name="mymodel")
response = sc.predict(gateway="ambassador", transport="grpc")
assert mock_grpc_predict_ambassador.call_count == 1