def test_deploy_yaml():
rt = SeldonDeployRuntime(
host="http://34.78.44.92/seldon-deploy/api/v1alpha1",
user="******",
oidc_server="https://34.78.44.92/auth/realms/deploy-realm",
password="******",
oidc_client_id="sd-api",
verify_ssl=False,
)
options = RuntimeOptions(
runtime="tempo.seldon.SeldonKubernetesRuntime",
k8s_options=KubernetesOptions(namespace="seldon"),
ingress_options=IngressOptions(ssl=True, verify_ssl=False),
)
sklearn_model = Model(
name="test-iris-sklearn",
platform=ModelFramework.SKLearn,
uri="gs://seldon-models/sklearn/iris",
protocol=SeldonProtocol(),
runtime_options=options,
)
spec = rt.to_k8s_yaml(sklearn_model)
rtk = SeldonKubernetesRuntime()
expected = rtk.to_k8s_yaml(sklearn_model)
assert spec == expected
def test_deploy():
rt = SeldonDeployRuntime()
config = SeldonDeployConfig(
host="https://34.78.44.92/seldon-deploy/api/v1alpha1",
user="******",
password="******",
oidc_server="https://34.78.44.92/auth/realms/deploy-realm",
oidc_client_id="sd-api",
verify_ssl=False,
auth_type=SeldonDeployAuthType.oidc,
)
rt.authenticate(settings=config)
options = RuntimeOptions(
runtime="tempo.seldon.SeldonKubernetesRuntime",
k8s_options=KubernetesOptions(namespace="seldon"),
ingress_options=IngressOptions(ssl=True, verify_ssl=False),
)
sklearn_model = Model(
name="test-iris-sklearn",
platform=ModelFramework.SKLearn,
uri="gs://seldon-models/sklearn/iris",
protocol=SeldonProtocol(),
runtime_options=options,
)
rt.deploy(sklearn_model)
rt.wait_ready(sklearn_model)
print(sklearn_model(np.array([[4.9, 3.1, 1.5, 0.2]])))
rt.undeploy(sklearn_model)
def __init__(
self,
name: str,
pipeline_func: Callable[[Any], Any] = None,
protocol: Optional[Protocol] = None,
models: PipelineModels = None,
local_folder: str = None,
uri: str = None,
inputs: ModelDataType = None,
outputs: ModelDataType = None,
conda_env: str = None,
runtime_options: RuntimeOptions = RuntimeOptions(),
description: str = "",
):
super().__init__(
name=name,
# TODO: Should we unify names?
user_func=pipeline_func,
local_folder=local_folder,
uri=uri,
platform=ModelFramework.TempoPipeline,
inputs=inputs,
outputs=outputs,
conda_env=conda_env,
protocol=protocol,
runtime_options=runtime_options,
description=description,
)
if models is None:
models = PipelineModels()
self.models = models
def deploy(model: Any, options: RuntimeOptions = None) -> RemoteModel:
if options is None:
options = RuntimeOptions()
rt: Runtime = _get_runtime(options.runtime, options)
rm = RemoteModel(model, rt)
rm.deploy()
return rm
def sklearn_model() -> Model:
model_path = os.path.join(TESTDATA_PATH, "sklearn", "iris")
return Model(
name="test-iris-sklearn",
platform=ModelFramework.SKLearn,
uri="gs://seldon-models/sklearn/iris",
local_folder=model_path,
protocol=SeldonProtocol(),
runtime_options=RuntimeOptions(k8s_options=KubernetesOptions(namespace="production", replicas=1)),
)
def __init__(
self,
name: str,
protocol: Protocol = KFServingV2Protocol(),
local_folder: str = None,
uri: str = None,
platform: ModelFramework = None,
inputs: ModelDataType = None,
outputs: ModelDataType = None,
model_func: Callable[..., Any] = None,
conda_env: str = None,
runtime_options: RuntimeOptions = RuntimeOptions(),
description: str = "",
):
"""
Parameters
----------
name
Name of the pipeline. Needs to be Kubernetes compliant.
protocol
:class:`tempo.serve.protocol.Protocol`. Defaults to KFserving V2.
local_folder
Location of local artifacts.
uri
Location of remote artifacts.
platform
The :class:`tempo.serve.metadata.ModelFramework`
inputs
The input types.
outputs
The output types.
conda_env
The conda environment name to use. If not specified will look for conda.yaml in
local_folder or generate from current running environment.
runtime_options
The runtime options. Can be left empty and set when creating a runtime.
description
The description of the model
"""
super().__init__(
name,
# TODO: Should we unify names?
user_func=model_func,
local_folder=local_folder,
uri=uri,
platform=platform,
inputs=inputs,
outputs=outputs,
conda_env=conda_env,
protocol=protocol,
runtime_options=runtime_options,
description=description,
)
def test_seldon_model_yaml_auth():
m = Model(
name="test-iris-xgboost",
protocol=SeldonProtocol(),
platform=ModelFramework.XGBoost,
uri="gs://seldon-models/xgboost/iris",
local_folder="/tmp/model",
)
runtime = SeldonKubernetesRuntime(
runtime_options=RuntimeOptions(k8s_options=KubernetesOptions(authSecretName="auth"))
)
print(runtime.to_k8s_yaml(m))
def test_seldon_model_yaml_auth(expected):
m = Model(
name="test-iris-xgboost",
protocol=SeldonProtocol(),
platform=ModelFramework.XGBoost,
uri="gs://seldon-models/xgboost/iris",
local_folder="/tmp/model",
)
runtime = SeldonKubernetesRuntime(runtime_options=RuntimeOptions(
k8s_options=KubernetesOptions(authSecretName="auth")))
yaml_str = runtime.manifest(m)
yaml_obj = yaml.safe_load(yaml_str)
yaml_obj_expected = yaml.safe_load(expected)
del yaml_obj["metadata"]["annotations"]["seldon.io/tempo-model"]
assert yaml_obj == yaml_obj_expected
def get_container_spec(cls, model_details: ModelDetails, runtime_options: RuntimeOptions) -> dict:
mlserver_runtime = cls.MLServerRuntimes[model_details.platform]
env = {
"MLSERVER_HTTP_PORT": DefaultHTTPPort,
"MLSERVER_GRPC_PORT": DefaultGRPCPort,
"MLSERVER_MODEL_IMPLEMENTATION": mlserver_runtime,
"MLSERVER_MODEL_NAME": model_details.name,
"MLSERVER_MODEL_URI": DefaultModelsPath,
ENV_TEMPO_RUNTIME_OPTIONS: json.dumps(runtime_options.dict()),
}
return {
"image": cls.MLServerImage,
"environment": env,
}
def test_model_spec():
ms = ModelSpec(
model_details=ModelDetails(
name="test",
local_folder="",
uri="",
platform=ModelFramework.XGBoost,
inputs=ModelDataArgs(args=[ModelDataArg(ty=str)]),
outputs=ModelDataArgs(args=[]),
),
protocol=KFServingV2Protocol(),
runtime_options=RuntimeOptions(),
)
s = ms.json()
j = json.loads(s)
ms2 = ModelSpec(**j)
assert isinstance(ms2.protocol, KFServingV2Protocol)
assert ms2.model_details.inputs.args[0].ty == str
def test_tensorflow_spec():
md = ModelDetails(
name="test",
local_folder="",
uri="",
platform=ModelFramework.Tensorflow,
inputs=ModelDataArgs(args=[]),
outputs=ModelDataArgs(args=[]),
)
protocol = SeldonProtocol()
options = KubernetesOptions(namespace="production", replicas=1)
runtime_options = RuntimeOptions(k8s_options=options)
model_spec = ModelSpec(model_details=md,
protocol=protocol,
runtime_options=runtime_options)
spec = get_container_spec(model_spec)
assert "image" in spec
assert "command" in spec
def test_runtime_options(runtime, replicas):
r = RuntimeOptions(**runtime)
assert r.k8s_options.replicas == replicas
def test_create_k8s_runtime():
rto = RuntimeOptions()
rt = SeldonKubernetesRuntime(rto)
assert rt.runtime_options.runtime == "tempo.seldon.SeldonKubernetesRuntime"
def runtime(namespace: str) -> SeldonKubernetesRuntime:
return SeldonKubernetesRuntime(runtime_options=RuntimeOptions(
k8s_options=KubernetesOptions(namespace=namespace)))
def __init__(self, runtime_options: Optional[RuntimeOptions] = None):
if runtime_options is None:
runtime_options = RuntimeOptions()
runtime_options.runtime = "tempo.seldon.SeldonDockerRuntime"
super().__init__(runtime_options)
def runtime() -> SeldonDockerRuntime:
return SeldonDockerRuntime(RuntimeOptions())
def test_kubernetes_spec_pipeline():
details = ModelDetails(
name="inference-pipeline",
platform=ModelFramework.TempoPipeline,
uri="gs://seldon/tempo",
local_folder="",
inputs=ModelDataArgs(args=[]),
outputs=ModelDataArgs(args=[]),
)
options = KubernetesOptions(namespace="production", replicas=1)
protocol = KFServingV2Protocol()
runtime_options = RuntimeOptions(k8s_options=options)
model_spec = ModelSpec(model_details=details,
protocol=protocol,
runtime_options=runtime_options)
k8s_object = KubernetesSpec(model_spec)
expected = {
"apiVersion": "machinelearning.seldon.io/v1",
"kind": "SeldonDeployment",
"metadata": {
"annotations": {
"seldon.io/tempo-description":
"",
"seldon.io/tempo-model":
'{"model_details": '
'{"name": '
'"inference-pipeline", '
'"local_folder": "", '
'"uri": '
'"gs://seldon/tempo", '
'"platform": "tempo", '
'"inputs": {"args": '
'[]}, "outputs": '
'{"args": []}, '
'"description": ""}, '
'"protocol": '
'"tempo.kfserving.protocol.KFServingV2Protocol", '
'"runtime_options": '
'{"runtime": null, '
'"docker_options": '
'{"defaultRuntime": '
'"tempo.seldon.SeldonDockerRuntime"}, '
'"k8s_options": '
'{"replicas": 1, '
'"minReplicas": null, '
'"maxReplicas": null, '
'"authSecretName": '
"null, "
'"serviceAccountName": '
"null, "
'"defaultRuntime": '
'"tempo.seldon.SeldonKubernetesRuntime", '
'"namespace": '
'"production"}, '
'"ingress_options": '
'{"ingress": '
'"tempo.ingress.istio.IstioIngress", '
'"ssl": false, '
'"verify_ssl": true}}}',
},
"labels": {
"seldon.io/tempo": "true"
},
"name": "inference-pipeline",
"namespace": "production",
},
"spec": {
"protocol":
"kfserving",
"predictors": [{
"graph": {
"modelUri": details.uri,
"name": "inference-pipeline",
"type": "MODEL",
"implementation": "TEMPO_SERVER",
"serviceAccountName": "tempo-pipeline",
},
"name": "default",
"replicas": options.replicas,
}],
},
}
assert k8s_object.spec == expected
def __init__(self, runtime_options: Optional[RuntimeOptions] = None):
if runtime_options is None:
runtime_options = RuntimeOptions()
runtime_options.runtime = "tempo.kfserving.KFServingKubernetesRuntime"
super().__init__(runtime_options)