def get_feature_vector_by_uri(uri, project=None, update=True): """get feature vector object from db by uri""" db = mlrun.get_run_db() default_project = project or config.default_project # parse store://.. uri if mlrun.datastore.is_store_uri(uri): prefix, new_uri = mlrun.datastore.parse_store_uri(uri) if prefix != StorePrefix.FeatureVector: raise mlrun.errors.MLRunInvalidArgumentError( f"provided store uri ({uri}) does not represent a feature vector (prefix={prefix})" ) uri = new_uri project, name, tag, uid = parse_versioned_object_uri(uri, default_project) resource = mlrun.api.schemas.AuthorizationResourceTypes.feature_vector.to_resource_string( project, "feature-vector") if update: auth_input = AuthorizationVerificationInput( resource=resource, action=mlrun.api.schemas.AuthorizationAction.update) else: auth_input = AuthorizationVerificationInput( resource=resource, action=mlrun.api.schemas.AuthorizationAction.read) db.verify_authorization(auth_input) return db.get_feature_vector(name, project, tag, uid)
def _init_endpoint_record(graph_server, model: V2ModelServer): logger.info("Initializing endpoint records") try: project, uri, tag, hash_key = parse_versioned_object_uri( graph_server.function_uri) if model.version: versioned_model_name = f"{model.name}:{model.version}" else: versioned_model_name = f"{model.name}:latest" model_endpoint = ModelEndpoint( metadata=ModelEndpointMetadata(project=project, labels=model.labels), spec=ModelEndpointSpec( function_uri=graph_server.function_uri, model=versioned_model_name, model_class=model.__class__.__name__, model_uri=model.model_path, stream_path=config.model_endpoint_monitoring.store_prefixes. default.format(project=project, kind="stream"), active=True, ), status=ModelEndpointStatus(), ) db = mlrun.get_run_db() db.create_or_patch_model_endpoint( project=project, endpoint_id=model_endpoint.metadata.uid, model_endpoint=model_endpoint, ) except Exception as e: logger.error("Failed to create endpoint record", exc=e)
def from_string(cls, function_uri): project, uri, tag, hash_key = parse_versioned_object_uri(function_uri) return cls( project=project, function=uri, tag=tag or None, hash_key=hash_key or None, )
def parse_feature_set_uri(uri, project=None): """get feature set object from db by uri""" default_project = project or config.default_project # parse store://.. uri if mlrun.datastore.is_store_uri(uri): prefix, new_uri = mlrun.datastore.parse_store_uri(uri) if prefix != StorePrefix.FeatureSet: raise mlrun.errors.MLRunInvalidArgumentError( f"provided store uri ({uri}) does not represent a feature set (prefix={prefix})" ) uri = new_uri return parse_versioned_object_uri(uri, default_project)
def get_feature_vector_by_uri(uri, project=None): """get feature vector object from db by uri""" db = mlrun.get_run_db() default_project = project or config.default_project # parse store://.. uri if mlrun.datastore.is_store_uri(uri): prefix, new_uri = mlrun.datastore.parse_store_uri(uri) if prefix != StorePrefix.FeatureVector: raise mlrun.errors.MLRunInvalidArgumentError( f"provided store uri ({uri}) does not represent a feature vector (prefix={prefix})" ) uri = new_uri project, name, tag, uid = parse_versioned_object_uri(uri, default_project) return db.get_feature_vector(name, project, tag, uid)
def test_job_file(): filename = f"{examples_path}/training.py" fn = code_to_function(filename=filename, kind="job") assert fn.kind == "job", "kind not set, test failed" assert fn.spec.build.functionSourceCode, "code not embedded" assert fn.spec.build.origin_filename == filename, "did not record filename" run = fn.run(workdir=str(examples_path), local=True) project, uri, tag, hash_key = parse_versioned_object_uri(run.spec.function) local_fn = get_run_db().get_function(uri, project, tag=tag, hash_key=hash_key) assert local_fn["spec"]["command"] == filename, "wrong command path" assert (local_fn["spec"]["build"]["functionSourceCode"] == fn.spec.build. functionSourceCode), "code was not copied to local function"
def _parse_submit_run_body(db_session: Session, auth_info: mlrun.api.schemas.AuthInfo, data): task = data.get("task") function_dict = data.get("function") function_url = data.get("functionUrl") if not function_url and task: function_url = get_in(task, "spec.function") if not (function_dict or function_url) or not task: log_and_raise( HTTPStatus.BAD_REQUEST.value, reason="bad JSON, need to include function/url and task objects", ) # TODO: block exec for function["kind"] in ["", "local] (must be a # remote/container runtime) if function_dict and not function_url: function = new_function(runtime=function_dict) else: if "://" in function_url: function = import_function(url=function_url, project=task.get("metadata", {}).get("project")) else: project, name, tag, hash_key = parse_versioned_object_uri( function_url) function_record = get_db().get_function(db_session, name, project, tag, hash_key) if not function_record: log_and_raise( HTTPStatus.NOT_FOUND.value, reason=f"runtime error: function {function_url} not found", ) function = new_function(runtime=function_record) if function_dict: # The purpose of the function dict is to enable the user to override configurations of the existing function # without modifying it - to do that we're creating a function object from the request function dict and # assign values from it to the main function object function = enrich_function_from_dict(function, function_dict) # if auth given in request ensure the function pod will have these auth env vars set, otherwise the job won't # be able to communicate with the api ensure_function_has_auth_set(function, auth_info) return function, task
def _parse_start_function_body(db_session, data): url = data.get("functionUrl") if not url: log_and_raise( HTTPStatus.BAD_REQUEST.value, reason="runtime error: functionUrl not specified", ) project, name, tag, hash_key = parse_versioned_object_uri(url) runtime = get_db().get_function(db_session, name, project, tag, hash_key) if not runtime: log_and_raise( HTTPStatus.BAD_REQUEST.value, reason=f"runtime error: function {url} not found", ) return new_function(runtime=runtime)
def _generate_function_and_task_from_submit_run_body( db_session: Session, auth_info: mlrun.api.schemas.AuthInfo, data): function_dict, function_url, task = parse_submit_run_body(data) # TODO: block exec for function["kind"] in ["", "local] (must be a # remote/container runtime) if function_dict and not function_url: function = new_function(runtime=function_dict) else: if "://" in function_url: function = import_function(url=function_url, project=task.get("metadata", {}).get("project")) else: project, name, tag, hash_key = parse_versioned_object_uri( function_url) function_record = get_db().get_function(db_session, name, project, tag, hash_key) if not function_record: log_and_raise( HTTPStatus.NOT_FOUND.value, reason=f"runtime error: function {function_url} not found", ) function = new_function(runtime=function_record) if function_dict: # The purpose of the function dict is to enable the user to override configurations of the existing function # without modifying it - to do that we're creating a function object from the request function dict and # assign values from it to the main function object function = enrich_function_from_dict(function, function_dict) # if auth given in request ensure the function pod will have these auth env vars set, otherwise the job won't # be able to communicate with the api ensure_function_has_auth_set(function, auth_info) # if this was triggered by the UI, we will need to attempt auto-mount based on auto-mount config and params passed # in the auth_info. If this was triggered by the SDK, then auto-mount was already attempted and will be skipped. try_perform_auto_mount(function, auth_info) # Validate function's service-account, based on allowed SAs for the project, if existing in a project-secret. process_function_service_account(function) return function, task
def _init_endpoint_record(graph_server, voting_ensemble: VotingEnsemble): logger.info("Initializing endpoint records") try: project, uri, tag, hash_key = parse_versioned_object_uri( graph_server.function_uri) if voting_ensemble.version: versioned_model_name = f"{voting_ensemble.name}:{voting_ensemble.version}" else: versioned_model_name = f"{voting_ensemble.name}:latest" model_endpoint = ModelEndpoint( metadata=ModelEndpointMetadata(project=project), spec=ModelEndpointSpec( function_uri=graph_server.function_uri, model=versioned_model_name, model_class=voting_ensemble.__class__.__name__, stream_path=config.model_endpoint_monitoring.store_prefixes. default.format(project=project, kind="stream"), active=True, ), status=ModelEndpointStatus( children=list(voting_ensemble.routes.keys())), ) db = mlrun.get_run_db() db.create_or_patch_model_endpoint( project=project, endpoint_id=model_endpoint.metadata.uid, model_endpoint=model_endpoint, ) except Exception as exc: logger.warning( "Failed creating model endpoint record", exc=exc, traceback=traceback.format_exc(), )
def _init_endpoint_record(context, model_logger: Optional[_ModelLogPusher]): if model_logger is None or isinstance(model_logger.output_stream, _DummyStream): return try: project, uri, tag, hash_key = parse_versioned_object_uri( model_logger.function_uri) if model_logger.model.version: model = f"{model_logger.model.name}:{model_logger.model.version}" else: model = model_logger.model.name model_endpoint = ModelEndpoint( metadata=ModelEndpointMetadata(project=project, labels=model_logger.model.labels), spec=ModelEndpointSpec( function_uri=model_logger.function_uri, model=model, model_class=model_logger.model.__class__.__name__, model_uri=model_logger.model.model_path, stream_path=model_logger.stream_path, active=True, ), status=ModelEndpointStatus(), ) db = mlrun.get_run_db() db.create_or_patch( project=project, endpoint_id=model_endpoint.metadata.uid, model_endpoint=model_endpoint, ) except Exception as e: logger.error("Failed to create endpoint record", exc=e)
def get_feature_vector_by_uri(uri, project=None): """get feature vector object from db by uri""" db = mlrun.get_run_db() default_project = project or config.default_project project, name, tag, uid = parse_versioned_object_uri(uri, default_project) return db.get_feature_vector(name, project, tag, uid)
def _parse_submit_run_body(db_session: Session, data): task = data.get("task") function_dict = data.get("function") function_url = data.get("functionUrl") if not function_url and task: function_url = get_in(task, "spec.function") if not (function_dict or function_url) or not task: log_and_raise( HTTPStatus.BAD_REQUEST.value, reason="bad JSON, need to include function/url and task objects", ) # TODO: block exec for function["kind"] in ["", "local] (must be a # remote/container runtime) if function_dict and not function_url: function = new_function(runtime=function_dict) else: if "://" in function_url: function = import_function(url=function_url) else: project, name, tag, hash_key = parse_versioned_object_uri( function_url) function_record = get_db().get_function(db_session, name, project, tag, hash_key) if not function_record: log_and_raise( HTTPStatus.NOT_FOUND.value, reason="runtime error: function {} not found".format( function_url), ) function = new_function(runtime=function_record) if function_dict: # The purpose of the function dict is to enable the user to override configurations of the existing function # without modifying it - to do that we're creating a function object from the request function dict and # assign values from it to the main function object override_function = new_function(runtime=function_dict, kind=function.kind) for attribute in [ "volumes", "volume_mounts", "env", "resources", "image_pull_policy", "replicas", ]: override_value = getattr(override_function.spec, attribute, None) if override_value: if attribute == "env": for env_dict in override_value: function.set_env(env_dict["name"], env_dict["value"]) elif attribute == "volumes": function.spec.update_vols_and_mounts( override_value, []) elif attribute == "volume_mounts": # volume mounts don't have a well defined identifier (like name for volume) so we can't merge, # only override function.spec.volume_mounts = override_value elif attribute == "resources": # don't override it there are limits and requests but both are empty if override_value.get("limits", {}) or override_value.get( "requests", {}): setattr(function.spec, attribute, override_value) else: setattr(function.spec, attribute, override_value) return function, task
def _init_endpoint_record(graph_server, voting_ensemble: VotingEnsemble): logger.info("Initializing endpoint records") endpoint_uid = None try: project, uri, tag, hash_key = parse_versioned_object_uri( graph_server.function_uri) if voting_ensemble.version: versioned_model_name = f"{voting_ensemble.name}:{voting_ensemble.version}" else: versioned_model_name = f"{voting_ensemble.name}:latest" children_uids = [] for _, c in voting_ensemble.routes.items(): if hasattr(c, "endpoint_uid"): children_uids.append(c.endpoint_uid) model_endpoint = ModelEndpoint( metadata=ModelEndpointMetadata(project=project), spec=ModelEndpointSpec( function_uri=graph_server.function_uri, model=versioned_model_name, model_class=voting_ensemble.__class__.__name__, stream_path=config.model_endpoint_monitoring.store_prefixes. default.format(project=project, kind="stream"), active=True, ), status=ModelEndpointStatus( children=list(voting_ensemble.routes.keys()), endpoint_type=EndpointType.ROUTER, children_uids=children_uids, ), ) endpoint_uid = model_endpoint.metadata.uid db = mlrun.get_run_db() db.create_or_patch_model_endpoint( project=project, endpoint_id=model_endpoint.metadata.uid, model_endpoint=model_endpoint, ) for model_endpoint in children_uids: # here to update that it is a node now current_endpoint = db.get_model_endpoint( project=project, endpoint_id=model_endpoint) current_endpoint.status.endpoint_type = EndpointType.LEAF_EP db.create_or_patch_model_endpoint( project=project, endpoint_id=model_endpoint, model_endpoint=current_endpoint, ) except Exception as exc: logger.warning( "Failed creating model endpoint record", exc=exc, traceback=traceback.format_exc(), ) return endpoint_uid
def get_db_function(project, key) -> mlrun.runtimes.BaseRuntime: project_instance, name, tag, hash_key = parse_versioned_object_uri( key, project.metadata.name ) runtime = mlrun.get_run_db().get_function(name, project_instance, tag, hash_key) return mlrun.new_function(runtime=runtime)