def test_set_traffic(integration_test_url, project_name, use_google_oauth):
merlin.set_url(integration_test_url, use_google_oauth=use_google_oauth)
merlin.set_project(project_name)
merlin.set_model("set-traffic-sample", ModelType.SKLEARN)
model_dir = "test/sklearn-model"
MODEL_FILE = "model.joblib"
undeploy_all_version()
with merlin.new_model_version() as v:
clf = svm.SVC(gamma='scale')
iris = load_iris()
X, y = iris.data, iris.target
clf.fit(X, y)
dump(clf, os.path.join(model_dir, MODEL_FILE))
# Upload the serialized model to MLP
merlin.log_model(model_dir=model_dir)
endpoint = merlin.deploy(v)
sleep(5)
resp = requests.post(f"{endpoint.url}", json=request_json)
assert resp.status_code == 200
assert resp.json() is not None
assert len(resp.json()['predictions']) == len(request_json['instances'])
# Undeploy deployed model version
merlin.undeploy(v)
sleep(5)
# Redeploy and set traffic
merlin.deploy(v)
endpoint = merlin.set_traffic({v: 100})
sleep(5)
resp = requests.post(f"{endpoint.url}", json=request_json)
assert resp.status_code == 200
assert resp.json() is not None
assert len(resp.json()['predictions']) == len(request_json['instances'])
# Try to undeploy serving model version. It must be fail
with pytest.raises(Exception):
assert merlin.undeploy(v)
# Undeploy other running model version endpoints
undeploy_all_version()
def deploy(env, model_name, model_type, model_dir, project, url, min_replica,
max_replica, cpu_request, memory_request):
merlin.set_url(url)
target_env = merlin.get_environment(env)
resource_request = target_env.default_resource_request
if min_replica is not None:
resource_request.min_replica = int(min_replica)
if max_replica is not None:
resource_request.max_replica = int(max_replica)
if cpu_request is not None:
resource_request.cpu_request = cpu_request
if memory_request is not None:
resource_request.memory_request = memory_request
merlin.set_project(project)
merlin.set_model(model_name, ModelType(model_type))
with merlin.new_model_version() as v:
merlin.log_model(model_dir=model_dir)
try:
endpoint = merlin.deploy(v, env, resource_request)
if endpoint:
print('Model deployed to {}'.format(endpoint))
except Exception as e:
print(e)
def test_tensorflow(integration_test_url, project_name, use_google_oauth):
merlin.set_url(integration_test_url, use_google_oauth=use_google_oauth)
merlin.set_project(project_name)
merlin.set_model("tensorflow-sample", ModelType.TENSORFLOW)
model_dir = "test/tensorflow-model"
undeploy_all_version()
with merlin.new_model_version() as v:
merlin.log_model(model_dir=model_dir)
endpoint = merlin.deploy(v)
request_json = {
"signature_name": "predict",
"instances": [
{"sepal_length": 2.8, "sepal_width": 1.0, "petal_length": 6.8,
"petal_width": 0.4},
{"sepal_length": 0.1, "sepal_width": 0.5, "petal_length": 1.8,
"petal_width": 2.4}
]
}
resp = requests.post(f"{endpoint.url}", json=request_json)
assert resp.status_code == 200
assert resp.json() is not None
assert len(resp.json()['predictions']) == len(request_json['instances'])
merlin.undeploy(v)
sleep(5)
resp = requests.post(f"{endpoint.url}", json=request_json)
assert resp.status_code == 404
def test_pyfunc_env_vars(integration_test_url, project_name, use_google_oauth):
merlin.set_url(integration_test_url, use_google_oauth=use_google_oauth)
merlin.set_project(project_name)
merlin.set_model("pyfunc-env-vars-sample", ModelType.PYFUNC)
undeploy_all_version()
with merlin.new_model_version() as v:
v.log_pyfunc_model(model_instance=EnvVarModel(),
conda_env="test/pyfunc/env.yaml",
code_dir=["test"],
artifacts={})
env_vars = {"WORKERS": "8", "ENV_VAR_1": "1", "ENV_VAR_2": "2"}
endpoint = merlin.deploy(v, env_vars=env_vars)
resp = requests.post(f"{endpoint.url}", json=request_json)
assert resp.status_code == 200
assert resp.json() is not None
assert resp.json()['workers'] == "8"
assert resp.json()['env_var_1'] == "1"
assert resp.json()['env_var_2'] == "2"
assert env_vars.items() <= endpoint.env_vars.items()
merlin.undeploy(v)
sleep(5)
resp = requests.post(f"{endpoint.url}", json=request_json)
assert resp.status_code == 404
def test_pyfunc(integration_test_url, project_name, use_google_oauth):
merlin.set_url(integration_test_url, use_google_oauth=use_google_oauth)
merlin.set_project(project_name)
merlin.set_model("pyfunc-sample", ModelType.PYFUNC)
undeploy_all_version()
with merlin.new_model_version() as v:
iris = load_iris()
y = iris['target']
X = iris['data']
xgb_path = train_xgboost_model(X, y)
sklearn_path = train_sklearn_model(X, y)
v.log_pyfunc_model(model_instance=EnsembleModel(),
conda_env="test/pyfunc/env.yaml",
code_dir=["test"],
artifacts={
"xgb_model": xgb_path,
"sklearn_model": sklearn_path
})
endpoint = merlin.deploy(v)
sleep(5)
resp = requests.post(f"{endpoint.url}", json=request_json)
assert resp.status_code == 200
assert resp.json() is not None
assert len(resp.json()['predictions']) == len(request_json['instances'])
merlin.undeploy(v)
sleep(5)
resp = requests.post(f"{endpoint.url}", json=request_json)
assert resp.status_code == 404
def test_sklearn(integration_test_url, project_name, use_google_oauth):
merlin.set_url(integration_test_url, use_google_oauth=use_google_oauth)
merlin.set_project(project_name)
merlin.set_model("sklearn-sample", ModelType.SKLEARN)
model_dir = "test/sklearn-model"
MODEL_FILE = "model.joblib"
undeploy_all_version()
with merlin.new_model_version() as v:
clf = svm.SVC(gamma='scale')
iris = load_iris()
X, y = iris.data, iris.target
clf.fit(X, y)
dump(clf, os.path.join(model_dir, MODEL_FILE))
# Upload the serialized model to MLP
merlin.log_model(model_dir=model_dir)
endpoint = merlin.deploy(v)
resp = requests.post(f"{endpoint.url}", json=request_json)
assert resp.status_code == 200
assert resp.json() is not None
assert len(resp.json()['predictions']) == len(request_json['instances'])
merlin.undeploy(v)
sleep(5)
resp = requests.post(f"{endpoint.url}", json=request_json)
assert resp.status_code == 404
def test_feast_enricher(integration_test_url, project_name, use_google_oauth):
merlin.set_url(integration_test_url, use_google_oauth=use_google_oauth)
merlin.set_project(project_name)
merlin.set_model("feast-enricher", ModelType.PYFUNC)
undeploy_all_version()
with merlin.new_model_version() as v:
v.log_pyfunc_model(
model_instance=EchoModel(),
conda_env="test/pyfunc/env.yaml",
code_dir=["test"],
artifacts={},
)
transformer_config_path = os.path.join("test/transformer",
"feast_enricher.yaml")
transformer = StandardTransformer(config_file=transformer_config_path,
enabled=True)
request_json = {"driver_id": "1000"}
endpoint = merlin.deploy(v, transformer=transformer)
resp = requests.post(f"{endpoint.url}", json=request_json)
assert resp.status_code == 200
assert resp.json() is not None
feast_features = resp.json()["feast_features"]
assert feast_features is not None
assert pd.DataFrame(feast_features) is not None
merlin.undeploy(v)
def test_trasformer_pytorch_logger(integration_test_url, project_name,
use_google_oauth):
merlin.set_url(integration_test_url, use_google_oauth=use_google_oauth)
merlin.set_project(project_name)
merlin.set_model("transformer-logger", ModelType.PYTORCH)
model_dir = "test/transformer"
undeploy_all_version()
resource_request = ResourceRequest(1, 1, "100m", "200Mi")
transformer = Transformer(
"gcr.io/kubeflow-ci/kfserving/image-transformer:latest",
resource_request=resource_request,
)
logger = Logger(
model=LoggerConfig(enabled=True, mode=LoggerMode.ALL),
transformer=LoggerConfig(enabled=True, mode=LoggerMode.ALL),
)
with merlin.new_model_version() as v:
merlin.log_pytorch_model(model_dir=model_dir)
endpoint = merlin.deploy(transformer=transformer, logger=logger)
assert endpoint.logger is not None
model_config = endpoint.logger.model
assert model_config is not None
assert model_config.enabled
assert model_config.mode == LoggerMode.ALL
transformer_config = endpoint.logger.transformer
assert transformer_config is not None
assert transformer_config.enabled
assert transformer_config.mode == LoggerMode.ALL
with open(os.path.join("test/transformer", "input.json"), "r") as f:
req = json.load(f)
sleep(5)
resp = requests.post(f"{endpoint.url}", json=req)
assert resp.status_code == 200
assert resp.json() is not None
assert len(resp.json()["predictions"]) == len(req["instances"])
model_endpoint = merlin.serve_traffic({endpoint: 100})
sleep(5)
resp = requests.post(f"{model_endpoint.url}", json=req)
assert resp.status_code == 200
assert resp.json() is not None
assert len(resp.json()["predictions"]) == len(req["instances"])
# Try to undeploy serving model version. It must be fail
with pytest.raises(Exception):
assert merlin.undeploy(v)
# Undeploy other running model version endpoints
undeploy_all_version()
def test_pytorch_logger(integration_test_url, project_name, use_google_oauth):
merlin.set_url(integration_test_url, use_google_oauth=use_google_oauth)
merlin.set_project(project_name)
merlin.set_model("pytorch-logger", ModelType.PYTORCH)
model_dir = "test/pytorch-model"
undeploy_all_version()
logger = Logger(model=LoggerConfig(enabled=True, mode=LoggerMode.REQUEST))
with merlin.new_model_version() as v:
merlin.log_pytorch_model(model_dir=model_dir)
endpoint = merlin.deploy(logger=logger)
model_config = endpoint.logger.model
assert model_config is not None
assert model_config.enabled
assert model_config.mode == LoggerMode.REQUEST
transformer_config = endpoint.logger.transformer
assert transformer_config is None
resp = requests.post(f"{endpoint.url}", json=request_json)
assert resp.status_code == 200
assert resp.json() is not None
assert len(resp.json()["predictions"]) == len(request_json["instances"])
undeploy_all_version()
def test_custom_model_with_artifact(integration_test_url, project_name,
use_google_oauth):
merlin.set_url(integration_test_url, use_google_oauth=use_google_oauth)
merlin.set_project(project_name)
merlin.set_model("custom-w-artifact", ModelType.CUSTOM)
undeploy_all_version()
resource_request = ResourceRequest(1, 1, "1", "1Gi")
model_dir = "test/custom-model"
BST_FILE = "model.bst"
iris = load_iris()
y = iris["target"]
X = iris["data"]
dtrain = xgb.DMatrix(X, label=y)
param = {
"max_depth": 6,
"eta": 0.1,
"silent": 1,
"nthread": 4,
"num_class": 10,
"objective": "multi:softmax",
}
xgb_model = xgb.train(params=param, dtrain=dtrain)
model_file = os.path.join((model_dir), BST_FILE)
xgb_model.save_model(model_file)
with merlin.new_model_version() as v:
v.log_custom_model(
image="ghcr.io/tiopramayudi/custom-predictor-go:v0.2",
model_dir=model_dir)
endpoint = merlin.deploy(v,
resource_request=resource_request,
env_vars={"MODEL_FILE_NAME": BST_FILE})
sleep(5)
resp = requests.post(f"{endpoint.url}", json=request_json)
assert resp.status_code == 200
assert resp.json() is not None
assert resp.json()["predictions"] is not None
model_endpoint = merlin.serve_traffic({endpoint: 100})
sleep(5)
resp = requests.post(f"{model_endpoint.url}", json=request_json)
assert resp.status_code == 200
assert resp.json() is not None
assert resp.json()["predictions"] is not None
# Try to undeploy serving model version. It must be fail
with pytest.raises(Exception):
assert merlin.undeploy(v)
# Undeploy other running model version endpoints
undeploy_all_version()
def test_standard_transformer_without_feast(integration_test_url, project_name,
use_google_oauth):
merlin.set_url(integration_test_url, use_google_oauth=use_google_oauth)
merlin.set_project(project_name)
merlin.set_model("std-transformer", ModelType.PYFUNC)
undeploy_all_version()
with merlin.new_model_version() as v:
v.log_pyfunc_model(
model_instance=EchoModel(),
conda_env="test/pyfunc/env.yaml",
code_dir=["test"],
artifacts={},
)
transformer_config_path = os.path.join(
"test/transformer", "standard_transformer_no_feast.yaml")
transformer = StandardTransformer(config_file=transformer_config_path,
enabled=True)
endpoint = merlin.deploy(v, transformer=transformer)
request_json = {
"drivers": [{
"id": 1,
"name": "driver-1",
"vehicle": "motorcycle",
"previous_vehicle": "suv",
"rating": 4
}, {
"id": 2,
"name": "driver-2",
"vehicle": "sedan",
"previous_vehicle": "mpv",
"rating": 3
}],
"customer": {
"id": 1111
},
}
resp = requests.post(f"{endpoint.url}", json=request_json)
assert resp.status_code == 200
assert resp.json() is not None
exp_resp = {
"instances": {
"columns": [
"customer_id", "name", "rank", "rating", "vehicle",
"previous_vehicle"
],
"data": [[1111, "driver-2", 2.5, 0.5, 2, 3],
[1111, "driver-1", -2.5, 0.75, 0, 1]],
}
}
assert resp.json()["instances"] == exp_resp["instances"]
merlin.undeploy(v)
def test_standard_transformer_with_feast(integration_test_url, project_name,
use_google_oauth):
merlin.set_url(integration_test_url, use_google_oauth=use_google_oauth)
merlin.set_project(project_name)
merlin.set_model("std-transformer-feast", ModelType.PYFUNC)
undeploy_all_version()
with merlin.new_model_version() as v:
v.log_pyfunc_model(
model_instance=EchoModel(),
conda_env="test/pyfunc/env.yaml",
code_dir=["test"],
artifacts={},
)
transformer_config_path = os.path.join(
"test/transformer", "standard_transformer_with_feast.yaml")
transformer = StandardTransformer(config_file=transformer_config_path,
enabled=True)
endpoint = merlin.deploy(v, transformer=transformer)
request_json = {
"drivers": [
{
"id": "1234",
"name": "driver-1"
},
{
"id": "5678",
"name": "driver-2"
},
],
"customer": {
"id": 1111
},
}
resp = requests.post(f"{endpoint.url}", json=request_json)
assert resp.status_code == 200
assert resp.json() is not None
exp_resp = {
"instances": {
"columns": [
"rank",
"driver_id",
"customer_id",
"merlin_test_driver_features:test_int32",
"merlin_test_driver_features:test_float",
],
"data": [[0, "1234", 1111, -1, 0], [1, "5678", 1111, -1, 0]],
}
}
assert resp.json()["instances"] == exp_resp["instances"]
merlin.undeploy(v)
def test_resource_request(integration_test_url, project_name,
use_google_oauth):
merlin.set_url(integration_test_url, use_google_oauth=use_google_oauth)
merlin.set_project(project_name)
merlin.set_model("resource-request", ModelType.XGBOOST)
model_dir = "test/xgboost-model"
BST_FILE = "model.bst"
envs = merlin.list_environment()
assert len(envs) >= 1
default_env = merlin.get_default_environment()
assert default_env is not None
undeploy_all_version()
with merlin.new_model_version() as v:
iris = load_iris()
y = iris['target']
X = iris['data']
dtrain = xgb.DMatrix(X, label=y)
param = {
'max_depth': 6,
'eta': 0.1,
'silent': 1,
'nthread': 4,
'num_class': 10,
'objective': 'multi:softmax'
}
xgb_model = xgb.train(params=param, dtrain=dtrain)
model_file = os.path.join(model_dir, BST_FILE)
xgb_model.save_model(model_file)
# Upload the serialized model to MLP
merlin.log_model(model_dir=model_dir)
resource_request = ResourceRequest(1, 1, "100m", "200Mi")
endpoint = merlin.deploy(v,
environment_name=default_env.name,
resource_request=resource_request)
sleep(5)
resp = requests.post(f"{endpoint.url}", json=request_json)
assert resp.status_code == 200
assert resp.json() is not None
assert len(resp.json()['predictions']) == len(request_json['instances'])
merlin.undeploy(v)
sleep(5)
resp = requests.post(f"{endpoint.url}", json=request_json)
assert resp.status_code == 404
def test_stop_serving_traffic(integration_test_url, project_name,
use_google_oauth):
merlin.set_url(integration_test_url, use_google_oauth=use_google_oauth)
merlin.set_project(project_name)
merlin.set_model("stop-serving-traffic", ModelType.SKLEARN)
model_dir = "test/sklearn-model"
MODEL_FILE = "model.joblib"
undeploy_all_version()
with merlin.new_model_version() as v:
clf = svm.SVC(gamma='scale')
iris = load_iris()
X, y = iris.data, iris.target
clf.fit(X, y)
dump(clf, os.path.join(model_dir, MODEL_FILE))
# Upload the serialized model to MLP
merlin.log_model(model_dir=model_dir)
endpoint = merlin.deploy(v)
sleep(5)
resp = requests.post(f"{endpoint.url}", json=request_json)
assert resp.status_code == 200
assert resp.json() is not None
assert len(resp.json()['predictions']) == len(request_json['instances'])
model_endpoint = merlin.serve_traffic({endpoint: 100})
sleep(5)
resp = requests.post(f"{model_endpoint.url}", json=request_json)
assert resp.status_code == 200
assert resp.json() is not None
assert len(resp.json()['predictions']) == len(request_json['instances'])
merlin.stop_serving_traffic(model_endpoint.environment_name)
endpoints = merlin.list_model_endpoints()
for endpoint in endpoints:
if endpoint.environment_name == model_endpoint.environment_name:
assert endpoint.status == Status.TERMINATED
# Undeploy other running model version endpoints
undeploy_all_version()
def test_pytorch(integration_test_url, project_name, use_google_oauth):
merlin.set_url(integration_test_url, use_google_oauth=use_google_oauth)
merlin.set_project(project_name)
merlin.set_model("pytorch-sample", ModelType.PYTORCH)
model_dir = "test/pytorch-model"
undeploy_all_version()
with merlin.new_model_version() as v:
merlin.log_pytorch_model(model_dir=model_dir)
endpoint = merlin.deploy()
resp = requests.post(f"{endpoint.url}", json=request_json)
assert resp.status_code == 200
assert resp.json() is not None
assert len(resp.json()["predictions"]) == len(request_json["instances"])
merlin.undeploy(v)
def test_xgboost(integration_test_url, project_name, use_google_oauth):
merlin.set_url(integration_test_url, use_google_oauth=use_google_oauth)
merlin.set_project(project_name)
merlin.set_model("xgboost-sample", ModelType.XGBOOST)
model_dir = "test/xgboost-model"
BST_FILE = "model.bst"
undeploy_all_version()
with merlin.new_model_version() as v:
iris = load_iris()
y = iris["target"]
X = iris["data"]
dtrain = xgb.DMatrix(X, label=y)
param = {
"max_depth": 6,
"eta": 0.1,
"silent": 1,
"nthread": 4,
"num_class": 10,
"objective": "multi:softmax",
}
xgb_model = xgb.train(params=param, dtrain=dtrain)
model_file = os.path.join(model_dir, BST_FILE)
xgb_model.save_model(model_file)
# Upload the serialized model to MLP
merlin.log_model(model_dir=model_dir)
resource_request = ResourceRequest(1, 1, "100m", "200Mi")
endpoint = merlin.deploy(v, resource_request=resource_request)
resp = requests.post(f"{endpoint.url}", json=request_json)
assert resp.status_code == 200
assert resp.json() is not None
assert len(resp.json()["predictions"]) == len(request_json["instances"])
merlin.undeploy(v)
def test_custom_model_without_artifact(integration_test_url, project_name,
use_google_oauth):
merlin.set_url(integration_test_url, use_google_oauth=use_google_oauth)
merlin.set_project(project_name)
merlin.set_model("custom-wo-artifact", ModelType.CUSTOM)
undeploy_all_version()
resource_request = ResourceRequest(1, 1, "1", "1Gi")
with merlin.new_model_version() as v:
v.log_custom_model(image="ghcr.io/tiopramayudi/custom-predictor:v0.2")
endpoint = merlin.deploy(v, resource_request=resource_request)
with open(os.path.join("test/custom-model", "input.json"), "r") as f:
req = json.load(f)
sleep(5)
resp = requests.post(f"{endpoint.url}", json=req)
assert resp.status_code == 200
assert resp.json() is not None
assert resp.json()["predictions"] is not None
model_endpoint = merlin.serve_traffic({endpoint: 100})
sleep(5)
resp = requests.post(f"{model_endpoint.url}", json=req)
assert resp.status_code == 200
assert resp.json() is not None
assert resp.json()["predictions"] is not None
# Try to undeploy serving model version. It must be fail
with pytest.raises(Exception):
assert merlin.undeploy(v)
# Undeploy other running model version endpoints
undeploy_all_version()
def test_pyfunc_old_infer(integration_test_url, project_name, use_google_oauth):
merlin.set_url(integration_test_url, use_google_oauth=use_google_oauth)
merlin.set_project(project_name)
merlin.set_model("pyfunc-old-infer-sample", ModelType.PYFUNC)
undeploy_all_version()
with merlin.new_model_version() as v:
v.log_pyfunc_model(model_instance=OldInferModel(),
conda_env="test/pyfunc/env.yaml",
code_dir=["test"],
artifacts={})
endpoint = merlin.deploy(v)
resp = requests.post(f"{endpoint.url}", json=request_json)
assert resp.status_code == 200
assert resp.json()['instances'] == request_json['instances']
merlin.undeploy(v)
sleep(5)
resp = requests.post(f"{endpoint.url}", json=request_json)
assert resp.status_code == 404
def test_standard_transformer_with_multiple_feast_with_source(
integration_test_url,
project_name,
use_google_oauth,
feast_serving_bigtable_url,
):
merlin.set_url(integration_test_url, use_google_oauth=use_google_oauth)
merlin.set_project(project_name)
merlin.set_model("std-trf-feasts-source", ModelType.PYFUNC)
undeploy_all_version()
with merlin.new_model_version() as v:
v.log_pyfunc_model(
model_instance=EchoModel(),
conda_env="test/pyfunc/env.yaml",
code_dir=["test"],
artifacts={},
)
config_template_file_path = os.path.join(
"test/transformer", "standard_transformer_feast_with_source.yaml.tmpl")
config_file_path = os.path.join(
"test/transformer", "standard_transformer_multiple_feast.yaml")
from string import Template
config_template_file = open(config_template_file_path, "rt")
t = Template(config_template_file.read())
rendered_config = t.substitute({
"feast_serving_bigtable_url":
feast_serving_bigtable_url,
})
config_file = open(config_file_path, "wt")
config_file.write(rendered_config)
config_file.close()
env_vars = {
"FEAST_REDIS_DIRECT_STORAGE_ENABLED": True,
"FEAST_REDIS_POOL_SIZE": 1,
"FEAST_BIGTABLE_DIRECT_STORAGE_ENABLED": True,
"FEAST_BIGTABLE_POOL_SIZE": 1,
"FEAST_BIGTABLE_KEEP_ALIVE_INTERVAL": "2m",
"FEAST_BIGTABLE_KEEP_ALIVE_TIMEOUT": "15s"
}
transformer = StandardTransformer(config_file=config_file_path,
enabled=True,
env_vars=env_vars)
endpoint = merlin.deploy(v, transformer=transformer)
request_json = {
"drivers": [
{
"id": "driver_1",
"name": "driver-1"
},
{
"id": "driver_2",
"name": "driver-2"
},
],
"customer": {
"id": 1111
},
}
resp = requests.post(f"{endpoint.url}", json=request_json)
assert resp.status_code == 200
assert resp.json() is not None
exp_resp = {
"instances": {
"columns": [
"rank", "driver_id", "customer_id",
"merlin_test_redis_driver_features:completion_rate",
"merlin_test_redis_driver_features:cancellation_rate",
"merlin_test_bt_driver_features:rating"
],
"data": [[0, "driver_1", 1111, 0.85, 0.15, 4.2],
[1, "driver_2", 1111, 0.6, 0.4, 4.2]]
}
}
assert resp.json()["instances"] == exp_resp["instances"]
merlin.undeploy(v)