def _build_function(
db_session,
auth_info: mlrun.api.schemas.AuthInfo,
function,
with_mlrun,
skip_deployed,
mlrun_version_specifier,
):
fn = None
ready = None
try:
fn = new_function(runtime=function)
run_db = get_run_db_instance(db_session, auth_info.session)
fn.set_db_connection(run_db)
fn.save(versioned=False)
if fn.kind in RuntimeKinds.nuclio_runtimes():
mlrun.api.api.utils.ensure_function_has_auth_set(fn, auth_info)
deploy_nuclio_function(fn)
# deploy only start the process, the get status API is used to check readiness
ready = False
else:
ready = build_runtime(fn, with_mlrun, mlrun_version_specifier,
skip_deployed)
fn.save(versioned=True)
logger.info("Fn:\n %s", fn.to_yaml())
except Exception as err:
logger.error(traceback.format_exc())
log_and_raise(HTTPStatus.BAD_REQUEST.value,
reason=f"runtime error: {err}")
return fn, ready
def test_deploy_with_triggers(self, db: Session, client: TestClient):
function = self._generate_runtime("nuclio")
http_trigger = {
"workers": 2,
"port": 12345,
"host": "http://my.host",
"paths": ["/path/1", "/path/2"],
"secret": "my little secret",
"canary": 50,
}
v3io_trigger = {
"stream_path": "/container/and/path",
"name": "test_stream",
"group": "beatles",
"seek_to": "latest",
"shards": 42,
}
function.with_http(**http_trigger)
function.add_v3io_stream_trigger(**v3io_trigger)
deploy_nuclio_function(function)
self._assert_deploy_called_basic_config()
self._assert_triggers(http_trigger, v3io_trigger)
def test_deploy_with_v3io(self, db: Session, client: TestClient):
function = self._generate_runtime("nuclio")
local_path = "/local/path"
remote_path = "/container/and/path"
function.with_v3io(local_path, remote_path)
deploy_nuclio_function(function)
self._assert_deploy_called_basic_config()
self._assert_nuclio_v3io_mount(local_path, remote_path)
def test_deploy_with_node_selection(self, db: Session, client: TestClient):
mlconf.nuclio_version = "1.6.10"
function = self._generate_runtime("nuclio")
node_name = "some-node-name"
function.with_node_selection(node_name=node_name)
deploy_nuclio_function(function)
self._assert_deploy_called_basic_config()
self._assert_node_selections(function.spec,
expected_node_name=node_name)
function = self._generate_runtime("nuclio")
node_selector = {
"label-1": "val1",
"label-2": "val2",
}
mlconf.default_function_node_selector = base64.b64encode(
json.dumps(node_selector).encode("utf-8"))
function.with_node_selection(node_selector=node_selector)
deploy_nuclio_function(function)
self._assert_deploy_called_basic_config(call_count=2)
self._assert_node_selections(function.spec,
expected_node_selector=node_selector)
function = self._generate_runtime("nuclio")
node_selector = {
"label-3": "val3",
"label-4": "val4",
}
function.with_node_selection(node_selector=node_selector)
deploy_nuclio_function(function)
self._assert_deploy_called_basic_config(call_count=3)
self._assert_node_selections(function.spec,
expected_node_selector=node_selector)
function = self._generate_runtime("nuclio")
affinity = self._generate_affinity()
function.with_node_selection(affinity=affinity)
deploy_nuclio_function(function)
self._assert_deploy_called_basic_config(call_count=4)
self._assert_node_selections(function.spec, expected_affinity=affinity)
function = self._generate_runtime("nuclio")
function.with_node_selection(node_name, node_selector, affinity)
deploy_nuclio_function(function)
self._assert_deploy_called_basic_config(call_count=5)
self._assert_node_selections(
function.spec,
expected_node_name=node_name,
expected_node_selector=node_selector,
expected_affinity=affinity,
)
def test_deploy_function_with_labels(self, db: Session,
client: TestClient):
labels = {
"key": "value",
"key-2": "value-2",
}
function = self._generate_runtime("nuclio", labels)
deploy_nuclio_function(function)
self._assert_deploy_called_basic_config(expected_labels=labels)
def _remote_db_mock_function(func, with_mlrun):
deploy_nuclio_function(func)
return {
"data": {
"status": NuclioStatus(
state="ready",
nuclio_name=f"nuclio-{func.metadata.name}",
address="http://127.0.0.1:1234",
)
}
}
def _remote_db_mock_function(func, with_mlrun):
deploy_nuclio_function(func)
return {
"data": {
"status": NuclioStatus(
state="ready",
nuclio_name=f"nuclio-{func.metadata.name}",
address="http://127.0.0.1:1234",
external_invocation_urls=["http://somewhere-far-away.com"],
internal_invocation_urls=["http://127.0.0.1:1234"],
)
}
}
def test_deploy_with_priority_class_name(self, db: Session,
client: TestClient):
mlconf.nuclio_version = "1.5.20"
default_priority_class_name = "default-priority"
mlrun.mlconf.default_function_priority_class_name = default_priority_class_name
mlrun.mlconf.valid_function_priority_class_names = default_priority_class_name
function = self._generate_runtime("nuclio")
deploy_nuclio_function(function)
self._assert_deploy_called_basic_config()
args, _ = nuclio.deploy.deploy_config.call_args
deploy_spec = args[0]["spec"]
assert "priorityClassName" not in deploy_spec
mlconf.nuclio_version = "1.6.18"
mlrun.mlconf.valid_function_priority_class_names = ""
function = self._generate_runtime("nuclio")
deploy_nuclio_function(function)
self._assert_deploy_called_basic_config(call_count=2)
args, _ = nuclio.deploy.deploy_config.call_args
deploy_spec = args[0]["spec"]
assert "priorityClassName" not in deploy_spec
mlrun.mlconf.valid_function_priority_class_names = default_priority_class_name
function = self._generate_runtime("nuclio")
deploy_nuclio_function(function)
self._assert_deploy_called_basic_config(call_count=3)
args, _ = nuclio.deploy.deploy_config.call_args
deploy_spec = args[0]["spec"]
assert deploy_spec["priorityClassName"] == default_priority_class_name
function = self._generate_runtime()
medium_priority_class_name = "medium-priority"
mlrun.mlconf.valid_function_priority_class_names = medium_priority_class_name
mlconf.nuclio_version = "1.5.20"
with pytest.raises(mlrun.errors.MLRunIncompatibleVersionError):
function.with_priority_class(medium_priority_class_name)
mlconf.nuclio_version = "1.6.10"
with pytest.raises(mlrun.errors.MLRunIncompatibleVersionError):
function.with_priority_class(medium_priority_class_name)
mlconf.nuclio_version = "1.6.18"
function.with_priority_class(medium_priority_class_name)
deploy_nuclio_function(function)
self._assert_deploy_called_basic_config(call_count=4)
args, _ = nuclio.deploy.deploy_config.call_args
deploy_spec = args[0]["spec"]
assert deploy_spec["priorityClassName"] == medium_priority_class_name
def _build_function(db_session, function, with_mlrun):
fn = None
ready = None
try:
fn = new_function(runtime=function)
run_db = get_run_db_instance(db_session)
fn.set_db_connection(run_db)
fn.save(versioned=False)
if fn.kind in RuntimeKinds.nuclio_runtimes():
deploy_nuclio_function(fn)
# deploy only start the process, the get status API is used to check readiness
ready = False
else:
ready = build_runtime(fn, with_mlrun)
fn.save(versioned=True)
logger.info("Fn:\n %s", fn.to_yaml())
except Exception as err:
logger.error(traceback.format_exc())
log_and_raise(HTTPStatus.BAD_REQUEST.value,
reason="runtime error: {}".format(err))
return fn, ready
def deploy_model_monitoring_stream_processing(
project: str,
model_monitoring_access_key: str,
auto_info: mlrun.api.schemas.AuthInfo,
):
logger.info(
f"Checking deployment status for model monitoring stream processing function [{project}]"
)
try:
get_nuclio_deploy_status(
name="model-monitoring-stream", project=project, tag=""
)
logger.info(
f"Detected model monitoring stream processing function [{project}] already deployed"
)
return
except DeployError:
logger.info(
f"Deploying model monitoring stream processing function [{project}]"
)
fn = get_model_monitoring_stream_processing_function(project)
fn.metadata.project = project
stream_path = config.model_endpoint_monitoring.store_prefixes.default.format(
project=project, kind="stream"
)
fn.add_v3io_stream_trigger(
stream_path=stream_path, name="monitoring_stream_trigger"
)
fn.set_env("MODEL_MONITORING_ACCESS_KEY", model_monitoring_access_key)
fn.set_env("MLRUN_AUTH_SESSION", model_monitoring_access_key)
fn.set_env("MODEL_MONITORING_PARAMETERS", json.dumps({"project": project}))
fn.apply(mlrun.mount_v3io())
deploy_nuclio_function(fn, auth_info=auto_info)
def _serialize_and_deploy_nuclio_function(self, function):
# simulating sending to API - serialization through dict
function = function.from_dict(function.to_dict())
deploy_nuclio_function(function)
def test_deploy_basic_function(self, db: Session, client: TestClient):
function = self._generate_runtime("nuclio")
deploy_nuclio_function(function)
self._assert_deploy_called_basic_config()
def _build_function(
db_session,
auth_info: mlrun.api.schemas.AuthInfo,
function,
with_mlrun=True,
skip_deployed=False,
mlrun_version_specifier=None,
builder_env=None,
):
fn = None
ready = None
try:
fn = new_function(runtime=function)
except Exception as err:
logger.error(traceback.format_exc())
log_and_raise(HTTPStatus.BAD_REQUEST.value,
reason=f"runtime error: {err}")
try:
run_db = get_run_db_instance(db_session)
fn.set_db_connection(run_db)
fn.save(versioned=False)
if fn.kind in RuntimeKinds.nuclio_runtimes():
mlrun.api.api.utils.ensure_function_has_auth_set(fn, auth_info)
mlrun.api.api.utils.process_function_service_account(fn)
if fn.kind == RuntimeKinds.serving:
# Handle model monitoring
try:
if fn.spec.track_models:
logger.info(
"Tracking enabled, initializing model monitoring")
_init_serving_function_stream_args(fn=fn)
model_monitoring_access_key = _process_model_monitoring_secret(
db_session,
fn.metadata.project,
"MODEL_MONITORING_ACCESS_KEY",
)
_create_model_monitoring_stream(
project=fn.metadata.project)
mlrun.api.crud.ModelEndpoints(
).deploy_monitoring_functions(
project=fn.metadata.project,
model_monitoring_access_key=
model_monitoring_access_key,
db_session=db_session,
auth_info=auth_info,
)
except Exception as exc:
logger.warning(
"Failed deploying model monitoring infrastructure for the project",
project=fn.metadata.project,
exc=exc,
traceback=traceback.format_exc(),
)
deploy_nuclio_function(fn, auth_info=auth_info)
# deploy only start the process, the get status API is used to check readiness
ready = False
else:
ready = build_runtime(
auth_info,
fn,
with_mlrun,
mlrun_version_specifier,
skip_deployed,
builder_env=builder_env,
)
fn.save(versioned=True)
logger.info("Fn:\n %s", fn.to_yaml())
except Exception as err:
logger.error(traceback.format_exc())
log_and_raise(HTTPStatus.BAD_REQUEST.value,
reason=f"runtime error: {err}")
return fn, ready
