def test_rolling_update9(self, namespace, api_gateway):
if api_gateway == API_ISTIO_GATEWAY:
retry_run(
f"kubectl create -f ../resources/seldon-gateway.yaml -n {namespace}"
)
retry_run(
f"kubectl apply -f ../resources/graph1svc.json -n {namespace}")
wait_for_status("mymodel", namespace)
wait_for_rollout("mymodel", namespace, expected_deployments=2)
r = initial_rest_request("mymodel", namespace, endpoint=api_gateway)
assert r.status_code == 200
assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0]
retry_run(
f"kubectl apply -f ../resources/graph5svc.json -n {namespace}")
r = initial_rest_request("mymodel", namespace, endpoint=api_gateway)
assert r.status_code == 200
assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0]
i = 0
for i in range(50):
r = rest_request_ambassador("mymodel", namespace, api_gateway)
assert r.status_code == 200
res = r.json()
assert res["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0]
time.sleep(1)
assert i == 49
logging.warning("Success for test_rolling_update9")
run(f"kubectl delete -f ../resources/graph1svc.json -n {namespace}",
shell=True)
run(f"kubectl delete -f ../resources/graph5svc.json -n {namespace}",
shell=True)
def test_rolling_update5(self, namespace, api_gateway):
if api_gateway == API_ISTIO_GATEWAY:
retry_run(
f"kubectl create -f ../resources/seldon-gateway.yaml -n {namespace}"
)
retry_run(f"kubectl apply -f ../resources/graph1.json -n {namespace}")
wait_for_status("mymodel", namespace)
wait_for_rollout("mymodel", namespace)
logging.warning("Initial request")
r = initial_rest_request("mymodel", namespace, endpoint=api_gateway)
assert r.status_code == 200
assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0]
retry_run(f"kubectl apply -f ../resources/graph6.json -n {namespace}")
r = initial_rest_request("mymodel", namespace, endpoint=api_gateway)
assert r.status_code == 200
assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0]
i = 0
for i in range(50):
r = rest_request_ambassador("mymodel", namespace, api_gateway)
assert r.status_code == 200
res = r.json()
assert (res["data"]["tensor"]["values"] == [
1.0, 2.0, 3.0, 4.0
]) or (res["data"]["tensor"]["values"] == [5.0, 6.0, 7.0, 8.0])
if (not r.status_code == 200) or (res["data"]["tensor"]["values"]
== [5.0, 6.0, 7.0, 8.0]):
break
time.sleep(1)
assert i < 100
logging.warning("Success for test_rolling_update5")
run(f"kubectl delete -f ../resources/graph1.json -n {namespace}",
shell=True)
run(f"kubectl delete -f ../resources/graph6.json -n {namespace}",
shell=True)
def test_tracing_rest(namespace):
# Deploy model and check that is running
retry_run(
f"kubectl apply -f ../resources/graph-tracing.json -n {namespace}")
wait_for_status("mymodel", namespace)
wait_for_rollout("mymodel", namespace)
initial_rest_request("mymodel", namespace)
# We need the current pod name to find the right traces
deployment_names = get_deployment_names("mymodel", namespace)
deployment_name = deployment_names[0]
pod_names = get_pod_names(deployment_name, namespace)
pod_name = pod_names[0]
print("deployment name", deployment_name, "pod name", pod_name)
# The engine and the executor identify as different services and different
# operations against Jaeger. We need to consider both.
service = "executor"
operation = "predictions"
request_operation = "/predict"
# Get traces and assert their content
traces = get_traces(pod_name,
service,
operation,
_should_retry=_is_jaeger_syncing)
assert len(traces) == 1
trace = traces[0]
processes = trace["processes"]
assert len(processes) == 2
assert_trace(trace,
expected_operations=[operation, request_operation, "Predict"])
def test_tracing_rest(namespace):
# Deploy model and check that is running
retry_run(
f"kubectl apply -f ../resources/graph-tracing.json -n {namespace}")
wait_for_status("mymodel", namespace)
wait_for_rollout("mymodel", namespace)
initial_rest_request("mymodel", namespace)
# We need the current pod name to find the right traces
deployment_names = get_deployment_names("mymodel", namespace)
deployment_name = deployment_names[0]
pod_names = get_pod_names(deployment_name, namespace)
pod_name = pod_names[0]
# Get traces and assert their content
traces = get_traces(pod_name,
"executor",
"predictions",
_should_retry=_is_jaeger_syncing)
assert len(traces) == 1
trace = traces[0]
processes = trace["processes"]
assert len(processes) == 2
assert_trace(trace,
expected_operations=["predictions", "/predict", "Predict"])
def test_rolling_update_deployment(namespace, from_deployment, to_deployment):
from_file_path = to_resources_path(from_deployment)
retry_run(f"kubectl apply -f {from_file_path} -n {namespace}")
# Note that this is not yet parametrised!
wait_for_status("mymodel", namespace)
wait_for_rollout("mymodel", namespace)
logging.warning("Initial request")
r = initial_rest_request("mymodel", namespace)
assert r.status_code == 200
assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0]
to_file_path = to_resources_path(to_deployment)
retry_run(f"kubectl apply -f {to_file_path} -n {namespace}")
r = initial_rest_request("mymodel", namespace)
assert r.status_code == 200
assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0]
i = 0
for i in range(100):
r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR)
assert r.status_code == 200
res = r.json()
assert (res["data"]["tensor"]["values"] == [
1.0, 2.0, 3.0, 4.0
]) or (res["data"]["tensor"]["values"] == [5.0, 6.0, 7.0, 8.0])
if (not r.status_code == 200) or (res["data"]["tensor"]["values"]
== [5.0, 6.0, 7.0, 8.0]):
break
time.sleep(1)
assert i < 100
run(f"kubectl delete -f {from_file_path} -n {namespace}", shell=True)
run(f"kubectl delete -f {to_file_path} -n {namespace}", shell=True)
def test_rolling_update7(self, namespace, api_gateway):
retry_run(f"kubectl apply -f ../resources/graph1svc.json -n {namespace}")
wait_for_status("mymodel", namespace)
wait_for_rollout("mymodel", namespace, expected_deployments=2)
logging.warning("Initial request")
r = initial_rest_request("mymodel", namespace, endpoint=api_gateway)
assert r.status_code == 200
assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0]
retry_run(f"kubectl apply -f ../resources/graph3svc.json -n {namespace}")
r = initial_rest_request("mymodel", namespace, endpoint=api_gateway)
assert r.status_code == 200
assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0]
i = 0
for i in range(100):
r = rest_request_ambassador("mymodel", namespace, api_gateway)
assert r.status_code == 200
res = r.json()
assert (res["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0]) or (
res["data"]["tensor"]["values"] == [5.0, 6.0, 7.0, 8.0]
)
if (not r.status_code == 200) or (
res["data"]["tensor"]["values"] == [5.0, 6.0, 7.0, 8.0]
):
break
time.sleep(1)
assert i < 100
logging.warning("Success for test_rolling_update7")
run(f"kubectl delete -f ../resources/graph1svc.json -n {namespace}", shell=True)
run(f"kubectl delete -f ../resources/graph3svc.json -n {namespace}", shell=True)
def test_rolling_update9(self, api_gateway):
if api_gateway == API_AMBASSADOR:
ns_suffix = "ambas"
else:
ns_suffix = "istio"
namespace = "test-rolling-update-9" + ns_suffix
retry_run(f"kubectl create namespace {namespace}")
if api_gateway == API_ISTIO_GATEWAY:
retry_run(
f"kubectl create -f ../resources/seldon-gateway.yaml -n {namespace}"
)
retry_run(
f"kubectl apply -f ../resources/graph1svc.json -n {namespace}")
wait_for_status("mymodel", namespace)
wait_for_rollout("mymodel", namespace, expected_deployments=2)
r = initial_rest_request("mymodel", namespace, endpoint=api_gateway)
assert r.status_code == 200
assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0]
retry_run(
f"kubectl apply -f ../resources/graph5svc.json -n {namespace}")
r = initial_rest_request("mymodel", namespace, endpoint=api_gateway)
assert r.status_code == 200
assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0]
i = 0
for i in range(50):
r = rest_request_ambassador("mymodel", namespace, api_gateway)
assert r.status_code == 200
res = r.json()
assert (
"complex-model" in res["meta"]["requestPath"]
and res["meta"]["requestPath"]["complex-model"]
== "seldonio/fixed-model:0.1"
and res["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0]
) or (res["meta"]["requestPath"]["model1"]
== "seldonio/fixed-model:0.1"
and res["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0]
and res["meta"]["requestPath"]["model2"]
== "seldonio/fixed-model:0.1")
if (not r.status_code == 200) or ("model1"
in res["meta"]["requestPath"]):
break
time.sleep(1)
assert i < 100
logging.warning("Success for test_rolling_update9")
run(f"kubectl delete -f ../resources/graph1svc.json -n {namespace}",
shell=True)
run(f"kubectl delete -f ../resources/graph5svc.json -n {namespace}",
shell=True)
run(f"kubectl delete namespace {namespace}", shell=True)
def test_mlflow(self, namespace):
spec = "../../servers/mlflowserver/samples/elasticnet_wine.yaml"
retry_run(f"kubectl apply -f {spec} -n {namespace}")
wait_for_status("mlflow", namespace)
wait_for_rollout("mlflow", namespace)
time.sleep(1)
r = initial_rest_request(
"mlflow",
namespace,
data=[[6.3, 0.3, 0.34, 1.6, 0.049, 14, 132, 0.994, 3.3, 0.49,
9.5]],
dtype="ndarray",
names=[
"fixed acidity",
"volatile acidity",
"citric acid",
"residual sugar",
"chlorides",
"free sulfur dioxide",
"total sulfur dioxide",
"density",
"pH",
"sulphates",
"alcohol",
],
)
assert r.status_code == 200
run(f"kubectl delete -f {spec} -n {namespace}", shell=True)
def test_model_combiner_rest(self, namespace, s2i_python_version):
create_push_s2i_image(s2i_python_version, "one", "rest")
create_push_s2i_image(s2i_python_version, "two", "rest")
create_push_s2i_image(s2i_python_version, "combiner", "rest")
retry_run(
f"kubectl apply -f ../resources/tags_combiner_rest.json -n {namespace}"
)
wait_for_status("mymodel-tags-combiner", namespace)
wait_for_rollout("mymodel-tags-combiner", namespace)
r = initial_rest_request("mymodel-tags-combiner", namespace)
arr = np.array([[1, 2, 3]])
r = rest_request_ambassador("mymodel-tags-combiner",
namespace,
API_AMBASSADOR,
data=arr)
res = r.json()
logging.info(res)
assert r.status_code == 200
assert res["data"]["ndarray"] == [["model-1"], ["model-2"]]
assert res["meta"]["tags"] == {
"combiner": "yes",
"common": 2,
"model-1": "yes",
"model-2": "yes",
}
run(
f"kubectl delete -f ../resources/tags_combiner_rest.json -n {namespace}",
shell=True,
)
def test_sklearn(self, namespace):
spec = "../../servers/sklearnserver/samples/iris.yaml"
retry_run(f"kubectl apply -f {spec} -n {namespace}")
wait_for_status("sklearn", namespace)
wait_for_rollout("sklearn", namespace)
time.sleep(1)
logging.warning("Initial request")
r = initial_rest_request("sklearn",
namespace,
data=[[0.1, 0.2, 0.3, 0.4]],
dtype="ndarray")
assert r.status_code == 200
r = rest_request_ambassador("sklearn", namespace, method="metadata")
assert r.status_code == 200
res = r.json()
logging.warning(res)
assert res["name"] == "iris"
assert res["versions"] == ["iris/v1"]
r = grpc_request_ambassador("sklearn",
namespace,
data=np.array([[0.1, 0.2, 0.3, 0.4]]))
res = json.loads(json_format.MessageToJson(r))
logging.info(res)
logging.warning("Success for test_prepack_sklearn")
run(f"kubectl delete -f {spec} -n {namespace}", shell=True)
def test_combiner_rest(self, s2i_python_version):
namespace = "s2i-test-combiner-rest"
retry_run(f"kubectl create namespace {namespace}")
create_push_s2i_image(s2i_python_version, "model", "rest")
create_push_s2i_image(s2i_python_version, "combiner", "rest")
retry_run(
f"kubectl apply -f ../resources/s2i_python_combiner.json -n {namespace}"
)
wait_for_rollout("mycombiner-mycombiner-acc7c4d", namespace)
r = initial_rest_request("mycombiner", namespace)
arr = np.array([[1, 2, 3]])
r = rest_request_ambassador("mycombiner",
namespace,
API_AMBASSADOR,
data=arr)
res = r.json()
logging.warning(res)
assert r.status_code == 200
assert r.json()["data"]["tensor"]["shape"] == [1, 3]
assert r.json()["data"]["tensor"]["values"] == [3, 4, 5]
run(
f"kubectl delete -f ../resources/s2i_python_combiner.json -n {namespace}",
shell=True,
)
run(f"kubectl delete namespace {namespace}", shell=True)
def test_model_rest_non200(self, s2i_python_version):
namespace = "s2i-test-model-rest-non200"
retry_run(f"kubectl create namespace {namespace}")
create_push_s2i_image(s2i_python_version, "model", "rest_non200")
retry_run(
f"kubectl apply -f ../resources/s2i_python_model_non200.json -n {namespace}"
)
wait_for_rollout("mymodel-mymodel-4e3d66d", namespace)
r = initial_rest_request("mymodel", namespace)
arr = np.array([[1, 2, 3]])
r = rest_request_ambassador("mymodel",
namespace,
API_AMBASSADOR,
data=arr)
res = r.json()
logging.warning(res)
assert r.status_code == 200
assert r.json()["status"]["code"] == 400
assert r.json()["status"]["reason"] == "exception message"
assert r.json()["status"]["info"] == "exception caught"
assert r.json()["status"]["status"] == "FAILURE"
run(
f"kubectl delete -f ../resources/s2i_python_model_non200.json -n {namespace}",
shell=True,
)
run(f"kubectl delete namespace {namespace}", shell=True)
def test_model_rest_non200(self, namespace, s2i_python_version):
create_push_s2i_image(s2i_python_version, "model", "rest_non200")
retry_run(
f"kubectl apply -f ../resources/s2i_python_model_non200.json -n {namespace}"
)
wait_for_status("mymodel", namespace)
wait_for_rollout("mymodel", namespace)
r = initial_rest_request("mymodel", namespace)
arr = np.array([[1, 2, 3]])
r = rest_request_ambassador("mymodel",
namespace,
API_AMBASSADOR,
data=arr)
res = r.json()
logging.warning(res)
assert r.status_code == 500
assert r.json()["status"]["code"] == 500
assert (
r.json()["status"]["info"] ==
"Internal service call failed calling http://localhost:9000/predict status code 400"
)
run(
f"kubectl delete -f ../resources/s2i_python_model_non200.json -n {namespace}",
shell=True,
)
def test_abtest_model(self, namespace):
command = ("helm install myabtest ../../helm-charts/seldon-abtest "
f"--namespace {namespace}")
run(command, shell=True, check=True)
wait_for_status("myabtest", namespace)
wait_for_rollout("myabtest", namespace, expected_deployments=2)
initial_rest_request("myabtest", namespace)
logging.warning("Test Ambassador REST gateway")
r = rest_request_ambassador("myabtest", namespace, API_AMBASSADOR)
logging.warning(r.json())
assert r.status_code == 200
assert len(r.json()["data"]["tensor"]["values"]) == 1
logging.warning("Test Ambassador gRPC gateway")
logging.warning(
"WARNING SKIPPING FLAKY AMBASSADOR TEST UNTIL AMBASSADOR GRPC ISSUE FIXED.."
)
run("helm delete myabtest", shell=True)
def test_single_model(self, namespace):
run(
f"helm install mymodel ../../helm-charts/seldon-single-model --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace {namespace}",
shell=True,
check=True,
)
wait_for_status("mymodel", namespace)
wait_for_rollout("mymodel", namespace)
initial_rest_request("mymodel", namespace)
logging.warning("Test Ambassador REST gateway")
r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR)
logging.warning(r.json())
assert r.status_code == 200
assert len(r.json()["data"]["tensor"]["values"]) == 1
logging.warning("Test Ambassador gRPC gateway")
r = grpc_request_ambassador2("mymodel", namespace, API_AMBASSADOR)
logging.warning(r)
run(f"helm delete mymodel", shell=True)
def test_api_version(namespace, apiVersion):
command = ("helm install mymodel ../../helm-charts/seldon-single-model "
"--set oauth.key=oauth-key "
"--set oauth.secret=oauth-secret "
f"--set apiVersion={apiVersion} "
f"--namespace {namespace}")
run(command, shell=True, check=True)
wait_for_status("mymodel", namespace)
wait_for_rollout("mymodel", namespace)
initial_rest_request("mymodel", namespace)
r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR)
assert r.status_code == 200
assert len(r.json()["data"]["tensor"]["values"]) == 1
run(f"helm delete mymodel", shell=True)
def test_rolling_update8(self, api_gateway):
if api_gateway == API_AMBASSADOR:
ns_suffix = "ambas"
else:
ns_suffix = "istio"
namespace = "test-rolling-update-8" + ns_suffix
retry_run(f"kubectl create namespace {namespace}")
if api_gateway == API_ISTIO_GATEWAY:
retry_run(
f"kubectl create -f ../resources/seldon-gateway.yaml -n {namespace}"
)
retry_run(
f"kubectl apply -f ../resources/graph1svc.json -n {namespace}")
wait_for_rollout("mymodel-mymodel-svc-orch-8e2a24b", namespace)
wait_for_rollout("mymodel-mymodel-e2eb561", namespace)
r = initial_rest_request("mymodel", namespace, endpoint=api_gateway)
assert r.status_code == 200
assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0]
retry_run(
f"kubectl apply -f ../resources/graph4svc.json -n {namespace}")
r = initial_rest_request("mymodel", namespace, endpoint=api_gateway)
assert r.status_code == 200
assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0]
i = 0
for i in range(50):
r = rest_request_ambassador("mymodel", namespace, api_gateway)
assert r.status_code == 200
res = r.json()
assert res["meta"]["requestPath"][
"complex-model"] == "seldonio/fixed-model:0.1" and res["data"][
"tensor"]["values"] == [
1.0,
2.0,
3.0,
4.0,
]
time.sleep(1)
assert i == 49
logging.warning("Success for test_rolling_update8")
run(f"kubectl delete -f ../resources/graph1svc.json -n {namespace}",
shell=True)
run(f"kubectl delete -f ../resources/graph4svc.json -n {namespace}",
shell=True)
run(f"kubectl delete namespace {namespace}", shell=True)
def test_mab_model(self, namespace):
run(
f"helm install mymab ../../helm-charts/seldon-mab --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace {namespace}",
shell=True,
check=True,
)
wait_for_status("mymab", namespace)
wait_for_rollout("mymab", namespace, expected_deployments=3)
initial_rest_request("mymab", namespace)
logging.warning("Test Ambassador REST gateway")
r = rest_request_ambassador("mymab", namespace, API_AMBASSADOR)
logging.warning(r.json())
assert r.status_code == 200
assert len(r.json()["data"]["tensor"]["values"]) == 1
logging.warning("Test Ambassador gRPC gateway")
logging.warning(
"WARNING SKIPPING FLAKY AMBASSADOR TEST UNTIL AMBASSADOR GRPC ISSUE FIXED.."
)
run(f"helm delete mymab", shell=True)
def test_rolling_update_deployment(from_deployment, to_deployment):
from_name = clean_string(from_deployment)
to_name = clean_string(to_deployment)
namespace = f"test-rolling-update-{from_name}-{to_name}"
retry_run(f"kubectl create namespace {namespace}")
from_file_path = to_resources_path(from_deployment)
retry_run(f"kubectl apply -f {from_file_path} -n {namespace}")
# Note that this is not yet parametrised!
wait_for_rollout("mymodel-mymodel-e2eb561", namespace)
logging.warning("Initial request")
r = initial_rest_request("mymodel", namespace)
assert r.status_code == 200
assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0]
to_file_path = to_resources_path(to_deployment)
retry_run(f"kubectl apply -f {to_file_path} -n {namespace}")
r = initial_rest_request("mymodel", namespace)
assert r.status_code == 200
assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0]
i = 0
for i in range(100):
r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR)
assert r.status_code == 200
res = r.json()
assert (res["meta"]["requestPath"]["complex-model"]
== "seldonio/fixed-model:0.1"
and res["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0]
) or (res["meta"]["requestPath"]["complex-model"]
== "seldonio/fixed-model:0.2" and
res["data"]["tensor"]["values"] == [5.0, 6.0, 7.0, 8.0])
if (not r.status_code == 200) or (res["data"]["tensor"]["values"]
== [5.0, 6.0, 7.0, 8.0]):
break
time.sleep(1)
assert i < 100
run(f"kubectl delete -f {from_file_path} -n {namespace}", shell=True)
run(f"kubectl delete -f {to_file_path} -n {namespace}", shell=True)
run(f"kubectl delete namespace {namespace}", shell=True)
def test_model_rest(s2i_image, namespace):
deploy_model("mymodel", namespace=namespace, model_image=s2i_image)
wait_for_status("mymodel", namespace)
wait_for_rollout("mymodel", namespace)
r = initial_rest_request("mymodel", namespace)
assert r.status_code == 200
assert r.json()["data"]["tensor"]["shape"] == [1]
assert r.json()["data"]["tensor"]["values"] == [1.0]
def test_text_alibi_explainer(self, namespace):
spec = "../resources/movies-text-explainer.yaml"
retry_run(f"kubectl apply -f {spec} -n {namespace}")
wait_for_status("movie", namespace)
wait_for_rollout("movie", namespace, expected_deployments=2)
time.sleep(5)
logging.warning("Initial request")
r = initial_rest_request("movie",
namespace,
data=["This is test data"],
dtype="ndarray")
log_sdep_logs("movie", namespace)
assert r.status_code == 200
# First request most likely will fail because AnchorText explainer
# is creating the explainer on first request - we skip checking output
# of it, sleep for some time and then do the actual explanation request
# we use in the test
e = initial_rest_request(
"movie",
namespace,
data=["This is test data"],
dtype="ndarray",
method="explain",
predictor_name="movies-predictor",
)
log_sdep_logs("movie", namespace)
time.sleep(30)
e = initial_rest_request(
"movie",
namespace,
data=["This is test data"],
dtype="ndarray",
method="explain",
predictor_name="movies-predictor",
)
log_sdep_logs("movie", namespace)
assert e.status_code == 200
logging.warning("Success for test_prepack_sklearn")
run(f"kubectl delete -f {spec} -n {namespace}", shell=True)
def test_xss_header(namespace):
sdep_name = "mymodel"
sdep_path = to_resources_path("graph-echo.json")
retry_run(f"kubectl apply -f {sdep_path} -n {namespace}")
wait_for_status(sdep_name, namespace)
wait_for_rollout(sdep_name, namespace)
res = initial_rest_request(sdep_name, namespace)
assert "X-Content-Type-Options" in res.headers
assert res.headers["X-Content-Type-Options"] == "nosniff"
def test_openapi_sklearn(self, namespace):
spec = "../../servers/sklearnserver/samples/iris.yaml"
retry_run(f"kubectl apply -f {spec} -n {namespace}")
wait_for_status("sklearn", namespace)
wait_for_rollout("sklearn", namespace)
time.sleep(1)
logging.warning("Initial request")
r = initial_rest_request("sklearn", namespace, method="openapi_ui")
assert r.status_code == 200
content_type_header = r.headers.get("content-type")
assert "text/html" in content_type_header
r = initial_rest_request("sklearn", namespace, method="openapi_schema")
assert r.status_code == 200
openapi_schema = r.json()
assert "openapi" in openapi_schema
logging.warning("Success for test_openapi_sklearn")
run(f"kubectl delete -f {spec} -n {namespace}", shell=True)
def test_api_version(apiVersion):
version = apiVersion.split("/")[-1]
namespace = f"test-api-version-{version}"
retry_run(f"kubectl create namespace {namespace}")
command = ("helm install mymodel ../../helm-charts/seldon-single-model "
"--set oauth.key=oauth-key "
"--set oauth.secret=oauth-secret "
f"--set apiVersion={apiVersion} "
f"--namespace {namespace}")
run(command, shell=True, check=True)
wait_for_rollout(f"mymodel-mymodel-7cd068f", namespace)
initial_rest_request("mymodel", namespace)
r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR)
assert r.status_code == 200
assert len(r.json()["data"]["tensor"]["values"]) == 1
run(f"helm delete mymodel", shell=True)
run(f"kubectl delete namespace {namespace}", shell=True)
def test_abtest_model(self):
namespace = "test-abtest-model"
retry_run(f"kubectl create namespace {namespace}")
run(
f"helm install myabtest ../../helm-charts/seldon-abtest --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace {namespace}",
shell=True,
check=True,
)
wait_for_rollout("myabtest-myabtest-41de5b8", namespace)
wait_for_rollout("myabtest-myabtest-df66c5c", namespace)
initial_rest_request("myabtest", namespace)
logging.warning("Test Ambassador REST gateway")
r = rest_request_ambassador("myabtest", namespace, API_AMBASSADOR)
logging.warning(r.json())
assert r.status_code == 200
assert len(r.json()["data"]["tensor"]["values"]) == 1
logging.warning("Test Ambassador gRPC gateway")
logging.warning(
"WARNING SKIPPING FLAKY AMBASSADOR TEST UNTIL AMBASSADOR GRPC ISSUE FIXED.."
)
run(f"helm delete myabtest", shell=True)
run(f"kubectl delete namespace {namespace}", shell=True)
def test_xgboost(self, namespace):
spec = "../../servers/xgboostserver/samples/iris.yaml"
retry_run(f"kubectl apply -f {spec} -n {namespace}")
wait_for_status("xgboost", namespace)
wait_for_rollout("xgboost", namespace)
time.sleep(1)
logging.warning("Initial request")
r = initial_rest_request(
"xgboost", namespace, data=[[0.1, 0.2, 0.3, 0.4]], dtype="ndarray"
)
assert r.status_code == 200
logging.warning("Success for test_prepack_xgboost")
run(f"kubectl delete -f {spec} -n {namespace}", shell=True)
def test_text_alibi_explainer(self, namespace):
spec = "../resources/movies-text-explainer.yaml"
retry_run(f"kubectl apply -f {spec} -n {namespace}")
wait_for_status("movie", namespace)
wait_for_rollout("movie", namespace, expected_deployments=2)
time.sleep(5)
logging.warning("Initial request")
r = initial_rest_request("movie",
namespace,
data=["This is test data"],
dtype="ndarray")
assert r.status_code == 200
e = initial_rest_request(
"movie",
namespace,
data=["This is test data"],
dtype="ndarray",
method="explain",
predictor_name="movies-predictor",
)
assert e.status_code == 200
logging.warning("Success for test_prepack_sklearn")
run(f"kubectl delete -f {spec} -n {namespace}", shell=True)
def test_namespace_operator(self, namespace):
retry_run(
f"helm install seldon ../../helm-charts/seldon-core-operator --namespace {namespace} --set executor.enabled=true --set istio.enabled=true --set istio.gateway=seldon-gateway --set certManager.enabled=false --set crd.create=false --set singleNamespace=true"
)
retry_run(f"kubectl apply -f ../resources/graph1.json -n {namespace}")
wait_for_status("mymodel", namespace)
wait_for_rollout("mymodel", namespace)
logging.warning("Initial request")
r = initial_rest_request("mymodel", namespace, endpoint=API_AMBASSADOR)
assert r.status_code == 200
assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0]
logging.warning("Success for test_namespace_operator")
run(f"kubectl delete -f ../resources/graph1.json -n {namespace}", shell=True)
run(f"helm uninstall seldon -n {namespace}", shell=True)
def test_rolling_update3(self, namespace, api_gateway):
if api_gateway == API_ISTIO_GATEWAY:
retry_run(
f"kubectl create -f ../resources/seldon-gateway.yaml -n {namespace}"
)
retry_run(f"kubectl apply -f ../resources/graph1.json -n {namespace}")
wait_for_status("mymodel", namespace)
wait_for_rollout("mymodel", namespace)
logging.warning("Initial request")
r = initial_rest_request("mymodel", namespace, endpoint=api_gateway)
assert r.status_code == 200
assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0]
retry_run(f"kubectl apply -f ../resources/graph4.json -n {namespace}")
r = initial_rest_request("mymodel", namespace, endpoint=api_gateway)
assert r.status_code == 200
assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0]
i = 0
for i in range(50):
r = rest_request_ambassador("mymodel", namespace, api_gateway)
assert r.status_code == 200
res = r.json()
assert res["meta"]["requestPath"][
"complex-model"] == "seldonio/fixed-model:0.1" and res["data"][
"tensor"]["values"] == [
1.0,
2.0,
3.0,
4.0,
]
time.sleep(1)
assert i == 49
logging.warning("Success for test_rolling_update3")
run(f"kubectl delete -f ../resources/graph1.json -n {namespace}",
shell=True)
run(f"kubectl delete -f ../resources/graph4.json -n {namespace}",
shell=True)
def test_tfserving(self, namespace):
spec = "../../servers/tfserving/samples/mnist_rest.yaml"
retry_run(f"kubectl apply -f {spec} -n {namespace}")
wait_for_status("tfserving", namespace)
wait_for_rollout("tfserving", namespace)
time.sleep(1)
logging.warning("Initial request")
r = initial_rest_request(
"tfserving",
namespace,
data=[create_random_data(784)[1].tolist()],
dtype="ndarray",
)
assert r.status_code == 200
logging.warning("Success for test_prepack_tfserving")
run(f"kubectl delete -f {spec} -n {namespace}", shell=True)
