def _start_function(function, auth_info: mlrun.api.schemas.AuthInfo): db_session = mlrun.api.db.session.create_session() try: resource = runtime_resources_map.get(function.kind) if "start" not in resource: log_and_raise( HTTPStatus.BAD_REQUEST.value, reason="runtime error: 'start' not supported by this runtime", ) try: run_db = get_run_db_instance(db_session) function.set_db_connection(run_db) mlrun.api.api.utils.ensure_function_has_auth_set( function, auth_info) mlrun.api.api.utils.process_function_service_account(function) # resp = resource["start"](fn) # TODO: handle resp? resource["start"](function) function.save(versioned=False) logger.info("Fn:\n %s", function.to_yaml()) except Exception as err: logger.error(traceback.format_exc()) log_and_raise(HTTPStatus.BAD_REQUEST.value, reason=f"runtime error: {err}") finally: mlrun.api.db.session.close_session(db_session)
def _start_function(db_session, data): logger.info("start_function:\n{}".format(data)) url = data.get("functionUrl") if not url: log_and_raise(HTTPStatus.BAD_REQUEST, reason="runtime error: functionUrl not specified") project, name, tag, hash_key = parse_function_uri(url) runtime = get_db().get_function(db_session, name, project, tag, hash_key) if not runtime: log_and_raise( HTTPStatus.BAD_REQUEST, reason="runtime error: function {} not found".format(url)) fn = new_function(runtime=runtime) resource = runtime_resources_map.get(fn.kind) if "start" not in resource: log_and_raise( HTTPStatus.BAD_REQUEST, reason="runtime error: 'start' not supported by this runtime") try: run_db = get_run_db_instance(db_session) fn.set_db_connection(run_db) # resp = resource["start"](fn) # TODO: handle resp? resource["start"](fn) fn.save(versioned=False) logger.info("Fn:\n %s", fn.to_yaml()) except Exception as err: logger.error(traceback.format_exc()) log_and_raise(HTTPStatus.BAD_REQUEST, reason="runtime error: {}".format(err)) return fn
def _build_function( db_session, auth_info: mlrun.api.schemas.AuthInfo, function, with_mlrun, skip_deployed, mlrun_version_specifier, ): fn = None ready = None try: fn = new_function(runtime=function) run_db = get_run_db_instance(db_session, auth_info.session) fn.set_db_connection(run_db) fn.save(versioned=False) if fn.kind in RuntimeKinds.nuclio_runtimes(): mlrun.api.api.utils.ensure_function_has_auth_set(fn, auth_info) deploy_nuclio_function(fn) # deploy only start the process, the get status API is used to check readiness ready = False else: ready = build_runtime(fn, with_mlrun, mlrun_version_specifier, skip_deployed) fn.save(versioned=True) logger.info("Fn:\n %s", fn.to_yaml()) except Exception as err: logger.error(traceback.format_exc()) log_and_raise(HTTPStatus.BAD_REQUEST.value, reason=f"runtime error: {err}") return fn, ready
def deploy_model_monitoring_batch_processing( project: str, model_monitoring_access_key: str, db_session, auth_info: mlrun.api.schemas.AuthInfo, ): logger.info( f"Checking deployment status for model monitoring batch processing function [{project}]" ) function_list = get_db().list_functions( session=db_session, name="model-monitoring-batch", project=project ) if function_list: logger.info( f"Detected model monitoring batch processing function [{project}] already deployed" ) return logger.info(f"Deploying model monitoring batch processing function [{project}]") fn: KubejobRuntime = mlrun.import_function( f"hub://model_monitoring_batch:{config.model_endpoint_monitoring.batch_processing_function_branch}" ) fn.set_db_connection(get_run_db_instance(db_session)) fn.metadata.project = project fn.apply(mlrun.mount_v3io()) fn.set_env_from_secret( "MODEL_MONITORING_ACCESS_KEY", mlrun.api.utils.singletons.k8s.get_k8s().get_project_secret_name(project), Secrets().generate_model_monitoring_secret_key( "MODEL_MONITORING_ACCESS_KEY" ), ) # Needs to be a member of the project and have access to project data path fn.metadata.credentials.access_key = model_monitoring_access_key function_uri = fn.save(versioned=True) function_uri = function_uri.replace("db://", "") task = mlrun.new_task(name="model-monitoring-batch", project=project) task.spec.function = function_uri data = { "task": task.to_dict(), "schedule": "0 */1 * * *", } _submit_run(db_session=db_session, auth_info=auth_info, data=data)
def _build_function(db_session, function, with_mlrun): fn = None ready = None try: fn = new_function(runtime=function) run_db = get_run_db_instance(db_session) fn.set_db_connection(run_db) fn.save(versioned=False) ready = build_runtime(fn, with_mlrun) fn.save(versioned=False) logger.info("Fn:\n %s", fn.to_yaml()) except Exception as err: logger.error(traceback.format_exc()) log_and_raise(HTTPStatus.BAD_REQUEST, reason="runtime error: {}".format(err)) return fn, ready
def _build_function(db_session, function, with_mlrun): fn = None ready = None try: fn = new_function(runtime=function) run_db = get_run_db_instance(db_session) fn.set_db_connection(run_db) fn.save(versioned=False) if fn.kind in RuntimeKinds.nuclio_runtimes(): deploy_nuclio_function(fn) # deploy only start the process, the get status API is used to check readiness ready = False else: ready = build_runtime(fn, with_mlrun) fn.save(versioned=True) logger.info("Fn:\n %s", fn.to_yaml()) except Exception as err: logger.error(traceback.format_exc()) log_and_raise(HTTPStatus.BAD_REQUEST.value, reason="runtime error: {}".format(err)) return fn, ready
def _start_function(function): db_session = mlrun.api.db.session.create_session() try: resource = runtime_resources_map.get(function.kind) if "start" not in resource: log_and_raise( HTTPStatus.BAD_REQUEST.value, reason="runtime error: 'start' not supported by this runtime", ) try: run_db = get_run_db_instance(db_session) function.set_db_connection(run_db) # resp = resource["start"](fn) # TODO: handle resp? resource["start"](function) function.save(versioned=False) logger.info("Fn:\n %s", function.to_yaml()) except Exception as err: logger.error(traceback.format_exc()) log_and_raise(HTTPStatus.BAD_REQUEST.value, reason="runtime error: {}".format(err)) finally: mlrun.api.db.session.close_session(db_session)
def _build_function( db_session, auth_info: mlrun.api.schemas.AuthInfo, function, with_mlrun=True, skip_deployed=False, mlrun_version_specifier=None, builder_env=None, ): fn = None ready = None try: fn = new_function(runtime=function) except Exception as err: logger.error(traceback.format_exc()) log_and_raise(HTTPStatus.BAD_REQUEST.value, reason=f"runtime error: {err}") try: run_db = get_run_db_instance(db_session) fn.set_db_connection(run_db) fn.save(versioned=False) if fn.kind in RuntimeKinds.nuclio_runtimes(): mlrun.api.api.utils.ensure_function_has_auth_set(fn, auth_info) mlrun.api.api.utils.process_function_service_account(fn) if fn.kind == RuntimeKinds.serving: # Handle model monitoring try: if fn.spec.track_models: logger.info( "Tracking enabled, initializing model monitoring") _init_serving_function_stream_args(fn=fn) model_monitoring_access_key = _process_model_monitoring_secret( db_session, fn.metadata.project, "MODEL_MONITORING_ACCESS_KEY", ) _create_model_monitoring_stream( project=fn.metadata.project) mlrun.api.crud.ModelEndpoints( ).deploy_monitoring_functions( project=fn.metadata.project, model_monitoring_access_key= model_monitoring_access_key, db_session=db_session, auth_info=auth_info, ) except Exception as exc: logger.warning( "Failed deploying model monitoring infrastructure for the project", project=fn.metadata.project, exc=exc, traceback=traceback.format_exc(), ) deploy_nuclio_function(fn, auth_info=auth_info) # deploy only start the process, the get status API is used to check readiness ready = False else: ready = build_runtime( auth_info, fn, with_mlrun, mlrun_version_specifier, skip_deployed, builder_env=builder_env, ) fn.save(versioned=True) logger.info("Fn:\n %s", fn.to_yaml()) except Exception as err: logger.error(traceback.format_exc()) log_and_raise(HTTPStatus.BAD_REQUEST.value, reason=f"runtime error: {err}") return fn, ready