def test_namespace_update(namespace, seldon_version):
# Deploy test model
retry_run(f"kubectl apply -f ../resources/graph1.json -n {namespace}")
wait_for_status("mymodel", namespace)
wait_for_rollout("mymodel", namespace)
assert_model("mymodel", namespace, initial=True)
# Label namespace to deploy a single operator
retry_run(
f"kubectl label namespace {namespace} seldon.io/controller-id={namespace}"
)
def _install_namespace_scoped():
# Install on the current namespace
retry_run(
"helm install seldon "
"../../helm-charts/seldon-core-operator "
f"--namespace {namespace} "
"--set crd.create=false "
"--set singleNamespace=true "
"--wait",
attempts=2,
)
# Assert that model is still working under new namespaced version
wait_for_status("mymodel", namespace)
wait_for_rollout("mymodel", namespace)
assert_model_during_op(_install_namespace_scoped, "mymodel", namespace)
def test_rolling_deployment(namespace, api_gateway, from_deployment,
to_deployment, change):
if api_gateway == API_ISTIO_GATEWAY:
retry_run(
f"kubectl create -f ../resources/seldon-gateway.yaml -n {namespace}"
)
from_file_path = to_resources_path(from_deployment)
retry_run(f"kubectl apply -f {from_file_path} -n {namespace}")
wait_for_status("mymodel", namespace)
wait_for_rollout("mymodel", namespace)
assert_model("mymodel", namespace, initial=True, endpoint=api_gateway)
old_pod_name = get_pod_name_for_sdep("mymodel", namespace)[0]
to_file_path = to_resources_path(to_deployment)
def _update_model():
retry_run(f"kubectl apply -f {to_file_path} -n {namespace}")
if change:
wait_for_pod_shutdown(old_pod_name, namespace)
wait_for_status("mymodel", namespace)
time.sleep(2) # Wait a little after deployment marked Available
assert_model_during_op(_update_model,
"mymodel",
namespace,
endpoint=api_gateway)
delete_cmd = f"kubectl delete --ignore-not-found -n {namespace}"
run(f"{delete_cmd} -f {from_file_path}", shell=True)
run(f"{delete_cmd} -f {to_file_path}", shell=True)
def test_api_version(namespace, apiVersion):
command = ("helm install mymodel ../../helm-charts/seldon-single-model "
f"--set apiVersion={apiVersion} "
f"--set model.image=seldonio/fixed-model:0.1 "
f"--namespace {namespace}")
run(command, shell=True, check=True)
wait_for_status("mymodel", namespace)
wait_for_rollout("mymodel", namespace)
assert_model("mymodel", namespace, initial=True)
run("helm delete mymodel", shell=True)
def test_cluster_update(namespace, seldon_version):
# Deploy test model
retry_run(f"kubectl apply -f ../resources/graph1.json -n {namespace}")
wait_for_status("mymodel", namespace)
wait_for_rollout("mymodel", namespace)
assert_model("mymodel", namespace, initial=True)
# Upgrade to source code version cluster-wide.
def _upgrade_seldon():
retry_run(
"helm upgrade seldon "
"../../helm-charts/seldon-core-operator "
"--namespace seldon-system "
"--wait",
attempts=2,
)
assert_model_during_op(_upgrade_seldon, "mymodel", namespace)
def test_cluster_update(namespace, seldon_version):
# NOTE: if this fails we might need to rebuild the test (fixed) models to pick up latest python changes.
# run:
# `make kind_build_test_models`
# Deploy test model
retry_run(f"kubectl apply -f ../resources/graph1.json -n {namespace}")
wait_for_status("mymodel", namespace)
wait_for_rollout("mymodel", namespace)
assert_model("mymodel", namespace, initial=True)
# Upgrade to source code version cluster-wide.
def _upgrade_seldon():
retry_run(
"helm upgrade seldon "
"../../helm-charts/seldon-core-operator "
"--namespace seldon-system "
"--wait",
attempts=2,
)
assert_model_during_op(_upgrade_seldon, "mymodel", namespace)
def test_label_update(namespace, seldon_version):
# Deploy test model
retry_run(f"kubectl apply -f ../resources/graph1.json -n {namespace}")
wait_for_status("mymodel", namespace)
wait_for_rollout("mymodel", namespace)
assert_model("mymodel", namespace, initial=True)
controller_id = f"seldon-{namespace}"
def _install_label_scoped():
# TODO: We install the new controller on the same namespace
# but it's not necessary, since it will get targeted by
# controllerId
retry_run(
f"helm install {controller_id} "
"../../helm-charts/seldon-core-operator "
f"--namespace {namespace} "
"--set crd.create=false "
f"--set controllerId={controller_id} "
"--wait",
attempts=2,
)
# Label model to be served by new controller
retry_run(
"kubectl label sdep mymodel "
f"seldon.io/controller-id={controller_id} "
f"--namespace {namespace}"
)
# Assert that model is still working under new id-scoped operator
wait_for_status("mymodel", namespace)
wait_for_rollout("mymodel", namespace)
assert_model_during_op(_install_label_scoped, "mymodel", namespace)
# Delete all resources (webhooks, etc.) before deleting namespace
retry_run(f"helm delete {controller_id} --namespace {namespace}")
