def test_model_rest_non200(self, s2i_python_version): namespace = "s2i-test-model-rest-non200" retry_run(f"kubectl create namespace {namespace}") create_push_s2i_image(s2i_python_version, "model", "rest_non200") retry_run( f"kubectl apply -f ../resources/s2i_python_model_non200.json -n {namespace}" ) wait_for_rollout("mymodel-mymodel-4e3d66d", namespace) r = initial_rest_request("mymodel", namespace) arr = np.array([[1, 2, 3]]) r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR, data=arr) res = r.json() logging.warning(res) assert r.status_code == 200 assert r.json()["status"]["code"] == 400 assert r.json()["status"]["reason"] == "exception message" assert r.json()["status"]["info"] == "exception caught" assert r.json()["status"]["status"] == "FAILURE" run( f"kubectl delete -f ../resources/s2i_python_model_non200.json -n {namespace}", shell=True, ) run(f"kubectl delete namespace {namespace}", shell=True)
def test_rolling_update5(self, namespace, api_gateway): if api_gateway == API_ISTIO_GATEWAY: retry_run( f"kubectl create -f ../resources/seldon-gateway.yaml -n {namespace}" ) retry_run(f"kubectl apply -f ../resources/graph1.json -n {namespace}") wait_for_status("mymodel", namespace) wait_for_rollout("mymodel", namespace) logging.warning("Initial request") r = initial_rest_request("mymodel", namespace, endpoint=api_gateway) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] retry_run(f"kubectl apply -f ../resources/graph6.json -n {namespace}") r = initial_rest_request("mymodel", namespace, endpoint=api_gateway) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] i = 0 for i in range(50): r = rest_request_ambassador("mymodel", namespace, api_gateway) assert r.status_code == 200 res = r.json() assert (res["data"]["tensor"]["values"] == [ 1.0, 2.0, 3.0, 4.0 ]) or (res["data"]["tensor"]["values"] == [5.0, 6.0, 7.0, 8.0]) if (not r.status_code == 200) or (res["data"]["tensor"]["values"] == [5.0, 6.0, 7.0, 8.0]): break time.sleep(1) assert i < 100 logging.warning("Success for test_rolling_update5") run(f"kubectl delete -f ../resources/graph1.json -n {namespace}", shell=True) run(f"kubectl delete -f ../resources/graph6.json -n {namespace}", shell=True)
def test_model_combiner_rest(self, namespace, s2i_python_version): create_push_s2i_image(s2i_python_version, "one", "rest") create_push_s2i_image(s2i_python_version, "two", "rest") create_push_s2i_image(s2i_python_version, "combiner", "rest") retry_run( f"kubectl apply -f ../resources/tags_combiner_rest.json -n {namespace}" ) wait_for_status("mymodel-tags-combiner", namespace) wait_for_rollout("mymodel-tags-combiner", namespace) r = initial_rest_request("mymodel-tags-combiner", namespace) arr = np.array([[1, 2, 3]]) r = rest_request_ambassador( "mymodel-tags-combiner", namespace, API_AMBASSADOR, data=arr ) res = r.json() logging.info(res) assert r.status_code == 200 assert res["data"]["ndarray"] == [["model-1"], ["model-2"]] assert res["meta"]["tags"] == { "combiner": "yes", "common": 2, "model-1": "yes", "model-2": "yes", } run( f"kubectl delete -f ../resources/tags_combiner_rest.json -n {namespace}", shell=True, )
def test_combiner_rest(self, s2i_python_version): namespace = "s2i-test-combiner-rest" retry_run(f"kubectl create namespace {namespace}") create_push_s2i_image(s2i_python_version, "model", "rest") create_push_s2i_image(s2i_python_version, "combiner", "rest") retry_run( f"kubectl apply -f ../resources/s2i_python_combiner.json -n {namespace}" ) wait_for_status("mycombiner", namespace) wait_for_rollout("mycombiner", namespace) r = initial_rest_request("mycombiner", namespace) arr = np.array([[1, 2, 3]]) r = rest_request_ambassador("mycombiner", namespace, API_AMBASSADOR, data=arr) res = r.json() logging.warning(res) assert r.status_code == 200 assert r.json()["data"]["tensor"]["shape"] == [1, 3] assert r.json()["data"]["tensor"]["values"] == [3, 4, 5] run( f"kubectl delete -f ../resources/s2i_python_combiner.json -n {namespace}", shell=True, ) run(f"kubectl delete namespace {namespace}", shell=True)
def test_rolling_update7(self, namespace, api_gateway): retry_run(f"kubectl apply -f ../resources/graph1svc.json -n {namespace}") wait_for_status("mymodel", namespace) wait_for_rollout("mymodel", namespace, expected_deployments=2) logging.warning("Initial request") r = initial_rest_request("mymodel", namespace, endpoint=api_gateway) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] retry_run(f"kubectl apply -f ../resources/graph3svc.json -n {namespace}") r = initial_rest_request("mymodel", namespace, endpoint=api_gateway) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] i = 0 for i in range(100): r = rest_request_ambassador("mymodel", namespace, api_gateway) assert r.status_code == 200 res = r.json() assert (res["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0]) or ( res["data"]["tensor"]["values"] == [5.0, 6.0, 7.0, 8.0] ) if (not r.status_code == 200) or ( res["data"]["tensor"]["values"] == [5.0, 6.0, 7.0, 8.0] ): break time.sleep(1) assert i < 100 logging.warning("Success for test_rolling_update7") run(f"kubectl delete -f ../resources/graph1svc.json -n {namespace}", shell=True) run(f"kubectl delete -f ../resources/graph3svc.json -n {namespace}", shell=True)
def test_rolling_update9(self, namespace, api_gateway): if api_gateway == API_ISTIO_GATEWAY: retry_run( f"kubectl create -f ../resources/seldon-gateway.yaml -n {namespace}" ) retry_run( f"kubectl apply -f ../resources/graph1svc.json -n {namespace}") wait_for_status("mymodel", namespace) wait_for_rollout("mymodel", namespace, expected_deployments=2) r = initial_rest_request("mymodel", namespace, endpoint=api_gateway) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] retry_run( f"kubectl apply -f ../resources/graph5svc.json -n {namespace}") r = initial_rest_request("mymodel", namespace, endpoint=api_gateway) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] i = 0 for i in range(50): r = rest_request_ambassador("mymodel", namespace, api_gateway) assert r.status_code == 200 res = r.json() assert res["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] time.sleep(1) assert i == 49 logging.warning("Success for test_rolling_update9") run(f"kubectl delete -f ../resources/graph1svc.json -n {namespace}", shell=True) run(f"kubectl delete -f ../resources/graph5svc.json -n {namespace}", shell=True)
def test_manifestmodelmetadata_rest(self, namespace, s2i_python_version): create_push_s2i_image(s2i_python_version, "manifestmetadata", "rest") retry_run( f"kubectl apply -f ../resources/metadata_manifestmetadata_rest.yaml -n {namespace}" ) wait_for_status("mymodel-manifestmetadata", namespace) wait_for_rollout("mymodel-manifestmetadata", namespace) r = initial_rest_request("mymodel-manifestmetadata", namespace) arr = np.array([[1, 2, 3]]) r = rest_request_ambassador("mymodel-manifestmetadata", namespace, API_AMBASSADOR, data=arr) res = r.json() logging.info(res) assert r.status_code == 200 r = rest_request_ambassador( "mymodel-manifestmetadata", namespace, method="metadata", model_name="my-model", ) assert r.status_code == 200 res = r.json() logging.warning(res) assert res == model_metadata r = rest_request_ambassador("mymodel-manifestmetadata", namespace, method="graph-metadata") assert r.status_code == 200 res = r.json() logging.warning(res) assert res == graph_metadata
def test_rolling_update9(self, api_gateway): if api_gateway == API_AMBASSADOR: ns_suffix = "ambas" else: ns_suffix = "istio" namespace = "test-rolling-update-9" + ns_suffix retry_run(f"kubectl create namespace {namespace}") if api_gateway == API_ISTIO_GATEWAY: retry_run( f"kubectl create -f ../resources/seldon-gateway.yaml -n {namespace}" ) retry_run( f"kubectl apply -f ../resources/graph1svc.json -n {namespace}") wait_for_status("mymodel", namespace) wait_for_rollout("mymodel", namespace, expected_deployments=2) r = initial_rest_request("mymodel", namespace, endpoint=api_gateway) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] retry_run( f"kubectl apply -f ../resources/graph5svc.json -n {namespace}") r = initial_rest_request("mymodel", namespace, endpoint=api_gateway) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] i = 0 for i in range(50): r = rest_request_ambassador("mymodel", namespace, api_gateway) assert r.status_code == 200 res = r.json() assert ( "complex-model" in res["meta"]["requestPath"] and res["meta"]["requestPath"]["complex-model"] == "seldonio/fixed-model:0.1" and res["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] ) or (res["meta"]["requestPath"]["model1"] == "seldonio/fixed-model:0.1" and res["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] and res["meta"]["requestPath"]["model2"] == "seldonio/fixed-model:0.1") if (not r.status_code == 200) or ("model1" in res["meta"]["requestPath"]): break time.sleep(1) assert i < 100 logging.warning("Success for test_rolling_update9") run(f"kubectl delete -f ../resources/graph1svc.json -n {namespace}", shell=True) run(f"kubectl delete -f ../resources/graph5svc.json -n {namespace}", shell=True) run(f"kubectl delete namespace {namespace}", shell=True)
def test_abtest_model(self, namespace): command = ("helm install myabtest ../../helm-charts/seldon-abtest " f"--namespace {namespace}") run(command, shell=True, check=True) wait_for_status("myabtest", namespace) wait_for_rollout("myabtest", namespace, expected_deployments=2) initial_rest_request("myabtest", namespace) logging.warning("Test Ambassador REST gateway") r = rest_request_ambassador("myabtest", namespace, API_AMBASSADOR) logging.warning(r.json()) assert r.status_code == 200 assert len(r.json()["data"]["tensor"]["values"]) == 1 logging.warning("Test Ambassador gRPC gateway") logging.warning( "WARNING SKIPPING FLAKY AMBASSADOR TEST UNTIL AMBASSADOR GRPC ISSUE FIXED.." ) run("helm delete myabtest", shell=True)
def test_single_model(self, namespace): run( f"helm install mymodel ../../helm-charts/seldon-single-model --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace {namespace}", shell=True, check=True, ) wait_for_status("mymodel", namespace) wait_for_rollout("mymodel", namespace) initial_rest_request("mymodel", namespace) logging.warning("Test Ambassador REST gateway") r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) logging.warning(r.json()) assert r.status_code == 200 assert len(r.json()["data"]["tensor"]["values"]) == 1 logging.warning("Test Ambassador gRPC gateway") r = grpc_request_ambassador2("mymodel", namespace, API_AMBASSADOR) logging.warning(r) run(f"helm delete mymodel", shell=True)
def test_api_version(namespace, apiVersion): command = ("helm install mymodel ../../helm-charts/seldon-single-model " "--set oauth.key=oauth-key " "--set oauth.secret=oauth-secret " f"--set apiVersion={apiVersion} " f"--namespace {namespace}") run(command, shell=True, check=True) wait_for_status("mymodel", namespace) wait_for_rollout("mymodel", namespace) initial_rest_request("mymodel", namespace) r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) assert r.status_code == 200 assert len(r.json()["data"]["tensor"]["values"]) == 1 run(f"helm delete mymodel", shell=True)
def test_rolling_update8(self, api_gateway): if api_gateway == API_AMBASSADOR: ns_suffix = "ambas" else: ns_suffix = "istio" namespace = "test-rolling-update-8" + ns_suffix retry_run(f"kubectl create namespace {namespace}") if api_gateway == API_ISTIO_GATEWAY: retry_run( f"kubectl create -f ../resources/seldon-gateway.yaml -n {namespace}" ) retry_run( f"kubectl apply -f ../resources/graph1svc.json -n {namespace}") wait_for_rollout("mymodel-mymodel-svc-orch-8e2a24b", namespace) wait_for_rollout("mymodel-mymodel-e2eb561", namespace) r = initial_rest_request("mymodel", namespace, endpoint=api_gateway) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] retry_run( f"kubectl apply -f ../resources/graph4svc.json -n {namespace}") r = initial_rest_request("mymodel", namespace, endpoint=api_gateway) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] i = 0 for i in range(50): r = rest_request_ambassador("mymodel", namespace, api_gateway) assert r.status_code == 200 res = r.json() assert res["meta"]["requestPath"][ "complex-model"] == "seldonio/fixed-model:0.1" and res["data"][ "tensor"]["values"] == [ 1.0, 2.0, 3.0, 4.0, ] time.sleep(1) assert i == 49 logging.warning("Success for test_rolling_update8") run(f"kubectl delete -f ../resources/graph1svc.json -n {namespace}", shell=True) run(f"kubectl delete -f ../resources/graph4svc.json -n {namespace}", shell=True) run(f"kubectl delete namespace {namespace}", shell=True)
def test_rolling_update_deployment(from_deployment, to_deployment): from_name = clean_string(from_deployment) to_name = clean_string(to_deployment) namespace = f"test-rolling-update-{from_name}-{to_name}" retry_run(f"kubectl create namespace {namespace}") from_file_path = to_resources_path(from_deployment) retry_run(f"kubectl apply -f {from_file_path} -n {namespace}") # Note that this is not yet parametrised! wait_for_status("mymodel", namespace) wait_for_rollout("mymodel", namespace) logging.warning("Initial request") r = initial_rest_request("mymodel", namespace) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] to_file_path = to_resources_path(to_deployment) retry_run(f"kubectl apply -f {to_file_path} -n {namespace}") r = initial_rest_request("mymodel", namespace) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] i = 0 for i in range(100): r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) assert r.status_code == 200 res = r.json() assert (res["meta"]["requestPath"]["complex-model"] == "seldonio/fixed-model:0.1" and res["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] ) or (res["meta"]["requestPath"]["complex-model"] == "seldonio/fixed-model:0.2" and res["data"]["tensor"]["values"] == [5.0, 6.0, 7.0, 8.0]) if (not r.status_code == 200) or (res["data"]["tensor"]["values"] == [5.0, 6.0, 7.0, 8.0]): break time.sleep(1) assert i < 100 run(f"kubectl delete -f {from_file_path} -n {namespace}", shell=True) run(f"kubectl delete -f {to_file_path} -n {namespace}", shell=True) run(f"kubectl delete namespace {namespace}", shell=True)
def test_mab_model(self, namespace): run( f"helm install mymab ../../helm-charts/seldon-mab --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace {namespace}", shell=True, check=True, ) wait_for_status("mymab", namespace) wait_for_rollout("mymab", namespace, expected_deployments=3) initial_rest_request("mymab", namespace) logging.warning("Test Ambassador REST gateway") r = rest_request_ambassador("mymab", namespace, API_AMBASSADOR) logging.warning(r.json()) assert r.status_code == 200 assert len(r.json()["data"]["tensor"]["values"]) == 1 logging.warning("Test Ambassador gRPC gateway") logging.warning( "WARNING SKIPPING FLAKY AMBASSADOR TEST UNTIL AMBASSADOR GRPC ISSUE FIXED.." ) run(f"helm delete mymab", shell=True)
def test_api_version(apiVersion): version = apiVersion.split("/")[-1] namespace = f"test-api-version-{version}" retry_run(f"kubectl create namespace {namespace}") command = ("helm install mymodel ../../helm-charts/seldon-single-model " "--set oauth.key=oauth-key " "--set oauth.secret=oauth-secret " f"--set apiVersion={apiVersion} " f"--namespace {namespace}") run(command, shell=True, check=True) wait_for_rollout(f"mymodel-mymodel-7cd068f", namespace) initial_rest_request("mymodel", namespace) r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR) assert r.status_code == 200 assert len(r.json()["data"]["tensor"]["values"]) == 1 run(f"helm delete mymodel", shell=True) run(f"kubectl delete namespace {namespace}", shell=True)
def test_input_transformer(self, namespace, s2i_python_version): create_push_s2i_image(s2i_python_version, "transformer", "") retry_run( f"kubectl apply -f ../resources/s2i_python_transformer.json -n {namespace}" ) wait_for_status("mytrans", namespace) wait_for_rollout("mytrans", namespace) r = initial_rest_request("mytrans", namespace) arr = np.array([[1, 2, 3]]) r = rest_request_ambassador("mytrans", namespace, API_AMBASSADOR, data=arr) res = r.json() logging.warning(res) assert r.status_code == 200 assert r.json()["data"]["tensor"]["shape"] == [1, 3] assert r.json()["data"]["tensor"]["values"] == [2, 3, 4] run( f"kubectl delete -f ../resources/s2i_python_transformer.json -n {namespace}", shell=True, )
def test_abtest_model(self): namespace = "test-abtest-model" retry_run(f"kubectl create namespace {namespace}") run( f"helm install myabtest ../../helm-charts/seldon-abtest --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace {namespace}", shell=True, check=True, ) wait_for_rollout("myabtest-myabtest-41de5b8", namespace) wait_for_rollout("myabtest-myabtest-df66c5c", namespace) initial_rest_request("myabtest", namespace) logging.warning("Test Ambassador REST gateway") r = rest_request_ambassador("myabtest", namespace, API_AMBASSADOR) logging.warning(r.json()) assert r.status_code == 200 assert len(r.json()["data"]["tensor"]["values"]) == 1 logging.warning("Test Ambassador gRPC gateway") logging.warning( "WARNING SKIPPING FLAKY AMBASSADOR TEST UNTIL AMBASSADOR GRPC ISSUE FIXED.." ) run(f"helm delete myabtest", shell=True) run(f"kubectl delete namespace {namespace}", shell=True)
def test_xgboost(self, namespace): spec = "../../servers/xgboostserver/samples/iris.yaml" retry_run(f"kubectl apply -f {spec} -n {namespace}") wait_for_status("xgboost", namespace) wait_for_rollout("xgboost", namespace) time.sleep(1) logging.warning("Initial request") r = initial_rest_request( "xgboost", namespace, data=[[0.1, 0.2, 0.3, 0.4]], dtype="ndarray" ) assert r.status_code == 200 r = rest_request_ambassador("xgboost", namespace, method="metadata") assert r.status_code == 200 res = r.json() logging.warning(res) assert res["name"] == "xgboost-iris" assert res["versions"] == ["xgboost-iris/v1"] logging.warning("Success for test_prepack_xgboost") run(f"kubectl delete -f {spec} -n {namespace}", shell=True)
def test_mlflow(self, namespace): spec = "../../servers/mlflowserver/samples/elasticnet_wine.yaml" retry_run(f"kubectl apply -f {spec} -n {namespace}") wait_for_status("mlflow", namespace) wait_for_rollout("mlflow", namespace) time.sleep(1) r = initial_rest_request( "mlflow", namespace, data=[[6.3, 0.3, 0.34, 1.6, 0.049, 14, 132, 0.994, 3.3, 0.49, 9.5]], dtype="ndarray", names=[ "fixed acidity", "volatile acidity", "citric acid", "residual sugar", "chlorides", "free sulfur dioxide", "total sulfur dioxide", "density", "pH", "sulphates", "alcohol", ], ) assert r.status_code == 200 r = rest_request_ambassador("mlflow", namespace, method="metadata") assert r.status_code == 200 res = r.json() logging.warning(res) assert res["name"] == "mlflow-wines" assert res["versions"] == ["mlflow-wines/v1"] run(f"kubectl delete -f {spec} -n {namespace}", shell=True)
def test_rolling_update3(self, namespace, api_gateway): if api_gateway == API_ISTIO_GATEWAY: retry_run( f"kubectl create -f ../resources/seldon-gateway.yaml -n {namespace}" ) retry_run(f"kubectl apply -f ../resources/graph1.json -n {namespace}") wait_for_status("mymodel", namespace) wait_for_rollout("mymodel", namespace) logging.warning("Initial request") r = initial_rest_request("mymodel", namespace, endpoint=api_gateway) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] retry_run(f"kubectl apply -f ../resources/graph4.json -n {namespace}") r = initial_rest_request("mymodel", namespace, endpoint=api_gateway) assert r.status_code == 200 assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0] i = 0 for i in range(50): r = rest_request_ambassador("mymodel", namespace, api_gateway) assert r.status_code == 200 res = r.json() assert res["meta"]["requestPath"][ "complex-model"] == "seldonio/fixed-model:0.1" and res["data"][ "tensor"]["values"] == [ 1.0, 2.0, 3.0, 4.0, ] time.sleep(1) assert i == 49 logging.warning("Success for test_rolling_update3") run(f"kubectl delete -f ../resources/graph1.json -n {namespace}", shell=True) run(f"kubectl delete -f ../resources/graph4.json -n {namespace}", shell=True)