def _start_function(function, auth_info: mlrun.api.schemas.AuthInfo):
db_session = mlrun.api.db.session.create_session()
try:
resource = runtime_resources_map.get(function.kind)
if "start" not in resource:
log_and_raise(
HTTPStatus.BAD_REQUEST.value,
reason="runtime error: 'start' not supported by this runtime",
)
try:
run_db = get_run_db_instance(db_session)
function.set_db_connection(run_db)
mlrun.api.api.utils.ensure_function_has_auth_set(
function, auth_info)
mlrun.api.api.utils.process_function_service_account(function)
# resp = resource["start"](fn) # TODO: handle resp?
resource["start"](function)
function.save(versioned=False)
logger.info("Fn:\n %s", function.to_yaml())
except Exception as err:
logger.error(traceback.format_exc())
log_and_raise(HTTPStatus.BAD_REQUEST.value,
reason=f"runtime error: {err}")
finally:
mlrun.api.db.session.close_session(db_session)
def _start_function(db_session, data):
logger.info("start_function:\n{}".format(data))
url = data.get("functionUrl")
if not url:
log_and_raise(HTTPStatus.BAD_REQUEST,
reason="runtime error: functionUrl not specified")
project, name, tag, hash_key = parse_function_uri(url)
runtime = get_db().get_function(db_session, name, project, tag, hash_key)
if not runtime:
log_and_raise(
HTTPStatus.BAD_REQUEST,
reason="runtime error: function {} not found".format(url))
fn = new_function(runtime=runtime)
resource = runtime_resources_map.get(fn.kind)
if "start" not in resource:
log_and_raise(
HTTPStatus.BAD_REQUEST,
reason="runtime error: 'start' not supported by this runtime")
try:
run_db = get_run_db_instance(db_session)
fn.set_db_connection(run_db)
# resp = resource["start"](fn) # TODO: handle resp?
resource["start"](fn)
fn.save(versioned=False)
logger.info("Fn:\n %s", fn.to_yaml())
except Exception as err:
logger.error(traceback.format_exc())
log_and_raise(HTTPStatus.BAD_REQUEST,
reason="runtime error: {}".format(err))
return fn
def _build_function(
db_session,
auth_info: mlrun.api.schemas.AuthInfo,
function,
with_mlrun,
skip_deployed,
mlrun_version_specifier,
):
fn = None
ready = None
try:
fn = new_function(runtime=function)
run_db = get_run_db_instance(db_session, auth_info.session)
fn.set_db_connection(run_db)
fn.save(versioned=False)
if fn.kind in RuntimeKinds.nuclio_runtimes():
mlrun.api.api.utils.ensure_function_has_auth_set(fn, auth_info)
deploy_nuclio_function(fn)
# deploy only start the process, the get status API is used to check readiness
ready = False
else:
ready = build_runtime(fn, with_mlrun, mlrun_version_specifier,
skip_deployed)
fn.save(versioned=True)
logger.info("Fn:\n %s", fn.to_yaml())
except Exception as err:
logger.error(traceback.format_exc())
log_and_raise(HTTPStatus.BAD_REQUEST.value,
reason=f"runtime error: {err}")
return fn, ready
def deploy_model_monitoring_batch_processing(
project: str,
model_monitoring_access_key: str,
db_session,
auth_info: mlrun.api.schemas.AuthInfo,
):
logger.info(
f"Checking deployment status for model monitoring batch processing function [{project}]"
)
function_list = get_db().list_functions(
session=db_session, name="model-monitoring-batch", project=project
)
if function_list:
logger.info(
f"Detected model monitoring batch processing function [{project}] already deployed"
)
return
logger.info(f"Deploying model monitoring batch processing function [{project}]")
fn: KubejobRuntime = mlrun.import_function(
f"hub://model_monitoring_batch:{config.model_endpoint_monitoring.batch_processing_function_branch}"
)
fn.set_db_connection(get_run_db_instance(db_session))
fn.metadata.project = project
fn.apply(mlrun.mount_v3io())
fn.set_env_from_secret(
"MODEL_MONITORING_ACCESS_KEY",
mlrun.api.utils.singletons.k8s.get_k8s().get_project_secret_name(project),
Secrets().generate_model_monitoring_secret_key(
"MODEL_MONITORING_ACCESS_KEY"
),
)
# Needs to be a member of the project and have access to project data path
fn.metadata.credentials.access_key = model_monitoring_access_key
function_uri = fn.save(versioned=True)
function_uri = function_uri.replace("db://", "")
task = mlrun.new_task(name="model-monitoring-batch", project=project)
task.spec.function = function_uri
data = {
"task": task.to_dict(),
"schedule": "0 */1 * * *",
}
_submit_run(db_session=db_session, auth_info=auth_info, data=data)
def _build_function(db_session, function, with_mlrun):
fn = None
ready = None
try:
fn = new_function(runtime=function)
run_db = get_run_db_instance(db_session)
fn.set_db_connection(run_db)
fn.save(versioned=False)
ready = build_runtime(fn, with_mlrun)
fn.save(versioned=False)
logger.info("Fn:\n %s", fn.to_yaml())
except Exception as err:
logger.error(traceback.format_exc())
log_and_raise(HTTPStatus.BAD_REQUEST,
reason="runtime error: {}".format(err))
return fn, ready
def _build_function(db_session, function, with_mlrun):
fn = None
ready = None
try:
fn = new_function(runtime=function)
run_db = get_run_db_instance(db_session)
fn.set_db_connection(run_db)
fn.save(versioned=False)
if fn.kind in RuntimeKinds.nuclio_runtimes():
deploy_nuclio_function(fn)
# deploy only start the process, the get status API is used to check readiness
ready = False
else:
ready = build_runtime(fn, with_mlrun)
fn.save(versioned=True)
logger.info("Fn:\n %s", fn.to_yaml())
except Exception as err:
logger.error(traceback.format_exc())
log_and_raise(HTTPStatus.BAD_REQUEST.value,
reason="runtime error: {}".format(err))
return fn, ready
def _start_function(function):
db_session = mlrun.api.db.session.create_session()
try:
resource = runtime_resources_map.get(function.kind)
if "start" not in resource:
log_and_raise(
HTTPStatus.BAD_REQUEST.value,
reason="runtime error: 'start' not supported by this runtime",
)
try:
run_db = get_run_db_instance(db_session)
function.set_db_connection(run_db)
# resp = resource["start"](fn) # TODO: handle resp?
resource["start"](function)
function.save(versioned=False)
logger.info("Fn:\n %s", function.to_yaml())
except Exception as err:
logger.error(traceback.format_exc())
log_and_raise(HTTPStatus.BAD_REQUEST.value,
reason="runtime error: {}".format(err))
finally:
mlrun.api.db.session.close_session(db_session)
def _build_function(
db_session,
auth_info: mlrun.api.schemas.AuthInfo,
function,
with_mlrun=True,
skip_deployed=False,
mlrun_version_specifier=None,
builder_env=None,
):
fn = None
ready = None
try:
fn = new_function(runtime=function)
except Exception as err:
logger.error(traceback.format_exc())
log_and_raise(HTTPStatus.BAD_REQUEST.value,
reason=f"runtime error: {err}")
try:
run_db = get_run_db_instance(db_session)
fn.set_db_connection(run_db)
fn.save(versioned=False)
if fn.kind in RuntimeKinds.nuclio_runtimes():
mlrun.api.api.utils.ensure_function_has_auth_set(fn, auth_info)
mlrun.api.api.utils.process_function_service_account(fn)
if fn.kind == RuntimeKinds.serving:
# Handle model monitoring
try:
if fn.spec.track_models:
logger.info(
"Tracking enabled, initializing model monitoring")
_init_serving_function_stream_args(fn=fn)
model_monitoring_access_key = _process_model_monitoring_secret(
db_session,
fn.metadata.project,
"MODEL_MONITORING_ACCESS_KEY",
)
_create_model_monitoring_stream(
project=fn.metadata.project)
mlrun.api.crud.ModelEndpoints(
).deploy_monitoring_functions(
project=fn.metadata.project,
model_monitoring_access_key=
model_monitoring_access_key,
db_session=db_session,
auth_info=auth_info,
)
except Exception as exc:
logger.warning(
"Failed deploying model monitoring infrastructure for the project",
project=fn.metadata.project,
exc=exc,
traceback=traceback.format_exc(),
)
deploy_nuclio_function(fn, auth_info=auth_info)
# deploy only start the process, the get status API is used to check readiness
ready = False
else:
ready = build_runtime(
auth_info,
fn,
with_mlrun,
mlrun_version_specifier,
skip_deployed,
builder_env=builder_env,
)
fn.save(versioned=True)
logger.info("Fn:\n %s", fn.to_yaml())
except Exception as err:
logger.error(traceback.format_exc())
log_and_raise(HTTPStatus.BAD_REQUEST.value,
reason=f"runtime error: {err}")
return fn, ready
