def store_feature_set( project: str, name: str, reference: str, feature_set: schemas.FeatureSet, versioned: bool = True, auth_verifier: deps.AuthVerifierDep = Depends(deps.AuthVerifierDep), db_session: Session = Depends(deps.get_db_session), ): mlrun.api.utils.singletons.project_member.get_project_member().ensure_project( db_session, project, auth_info=auth_verifier.auth_info ) mlrun.api.utils.clients.opa.Client().query_project_resource_permissions( mlrun.api.schemas.AuthorizationResourceTypes.feature_set, project, name, mlrun.api.schemas.AuthorizationAction.store, auth_verifier.auth_info, ) tag, uid = parse_reference(reference) uid = mlrun.api.crud.FeatureStore().store_feature_set( db_session, project, name, feature_set, tag, uid, versioned, ) return mlrun.api.crud.FeatureStore().get_feature_set( db_session, project, feature_set.metadata.name, tag, uid, )
def patch_feature_vector( project: str, name: str, feature_vector_patch: dict, reference: str, patch_mode: schemas.PatchMode = Header( schemas.PatchMode.replace, alias=schemas.HeaderNames.patch_mode ), auth_verifier: deps.AuthVerifierDep = Depends(deps.AuthVerifierDep), db_session: Session = Depends(deps.get_db_session), ): mlrun.api.utils.clients.opa.Client().query_project_resource_permissions( mlrun.api.schemas.AuthorizationResourceTypes.feature_vector, project, name, mlrun.api.schemas.AuthorizationAction.update, auth_verifier.auth_info, ) _verify_feature_vector_features_permissions( auth_verifier.auth_info, project, feature_vector_patch ) tag, uid = parse_reference(reference) mlrun.api.crud.FeatureStore().patch_feature_vector( db_session, project, name, feature_vector_patch, tag, uid, patch_mode, ) return Response(status_code=HTTPStatus.OK.value)
def store_feature_vector( project: str, name: str, reference: str, feature_vector: schemas.FeatureVector, versioned: bool = True, auth_info: mlrun.api.schemas.AuthInfo = Depends( deps.authenticate_request), db_session: Session = Depends(deps.get_db_session), ): mlrun.api.utils.singletons.project_member.get_project_member( ).ensure_project(db_session, project, auth_info=auth_info) mlrun.api.utils.auth.verifier.AuthVerifier( ).query_project_resource_permissions( mlrun.api.schemas.AuthorizationResourceTypes.feature_vector, project, name, mlrun.api.schemas.AuthorizationAction.update, auth_info, ) _verify_feature_vector_features_permissions(auth_info, project, feature_vector.dict()) tag, uid = parse_reference(reference) uid = mlrun.api.crud.FeatureStore().store_feature_vector( db_session, project, name, feature_vector, tag, uid, versioned, ) return mlrun.api.crud.FeatureStore().get_feature_vector( db_session, project, name, tag, uid)
def patch_feature_set( project: str, name: str, feature_set_update: dict, reference: str, patch_mode: schemas.PatchMode = Header( schemas.PatchMode.replace, alias=schemas.HeaderNames.patch_mode), auth_info: mlrun.api.schemas.AuthInfo = Depends( deps.authenticate_request), db_session: Session = Depends(deps.get_db_session), ): mlrun.api.utils.auth.verifier.AuthVerifier( ).query_project_resource_permissions( mlrun.api.schemas.AuthorizationResourceTypes.feature_set, project, name, mlrun.api.schemas.AuthorizationAction.update, auth_info, ) tag, uid = parse_reference(reference) mlrun.api.crud.FeatureStore().patch_feature_set( db_session, project, name, feature_set_update, tag, uid, patch_mode, ) return Response(status_code=HTTPStatus.OK.value)
def ingest_feature_set( request: Request, project: str, name: str, reference: str, ingest_parameters: Optional[ schemas.FeatureSetIngestInput ] = schemas.FeatureSetIngestInput(), username: str = Header(None, alias="x-remote-user"), auth_verifier: deps.AuthVerifier = Depends(deps.AuthVerifier), db_session: Session = Depends(deps.get_db_session), ): tag, uid = parse_reference(reference) feature_set_record = get_db().get_feature_set(db_session, project, name, tag, uid) feature_set = mlrun.feature_store.FeatureSet.from_dict(feature_set_record.dict()) # Need to override the default rundb since we're in the server. feature_set._override_run_db(db_session, auth_verifier.auth_info.session) data_source = data_targets = None if ingest_parameters.source: data_source = DataSource.from_dict(ingest_parameters.source.dict()) if ingest_parameters.targets: data_targets = [ DataTargetBase.from_dict(data_target.dict()) for data_target in ingest_parameters.targets ] run_config = RunConfig() # Try to deduce whether the ingest job will need v3io mount, by analyzing the paths to the source and # targets. If it needs it, apply v3io mount to the run_config. Note that the access-key and username are # user-context parameters, we cannot use the api context. if _has_v3io_path(data_source, data_targets, feature_set): secrets = get_secrets(request) access_key = secrets.get("V3IO_ACCESS_KEY", None) if not access_key or not username: log_and_raise( HTTPStatus.BAD_REQUEST.value, reason="Request needs v3io access key and username in header", ) run_config = run_config.apply(v3io_cred(access_key=access_key, user=username)) infer_options = ingest_parameters.infer_options or InferOptions.default() run_params = ingest( feature_set, data_source, data_targets, infer_options=infer_options, return_df=False, run_config=run_config, ) # ingest may modify the feature-set contents, so returning the updated feature-set. result_feature_set = schemas.FeatureSet(**feature_set.to_dict()) return schemas.FeatureSetIngestOutput( feature_set=result_feature_set, run_object=run_params.to_dict() )
def get_feature_vector( project: str, name: str, reference: str, db_session: Session = Depends(deps.get_db_session), ): tag, uid = parse_reference(reference) return get_db().get_feature_vector(db_session, project, name, tag, uid)
def update_feature_set( project: str, name: str, feature_set_update: schemas.FeatureSetUpdate, reference: str, db_session: Session = Depends(deps.get_db_session), ): tag, uid = parse_reference(reference) get_db().update_feature_set(db_session, project, name, feature_set_update, tag, uid) return Response(status_code=HTTPStatus.OK.value)
def delete_feature_set( project: str, name: str, reference: str = None, db_session: Session = Depends(deps.get_db_session), ): tag = uid = None if reference: tag, uid = parse_reference(reference) get_db().delete_feature_set(db_session, project, name, tag, uid) return Response(status_code=HTTPStatus.NO_CONTENT.value)
def patch_feature_set( project: str, name: str, feature_set_update: dict, reference: str, patch_mode: schemas.PatchMode = Header( schemas.PatchMode.replace, alias=schemas.HeaderNames.patch_mode), db_session: Session = Depends(deps.get_db_session), ): tag, uid = parse_reference(reference) get_db().patch_feature_set(db_session, project, name, feature_set_update, tag, uid, patch_mode) return Response(status_code=HTTPStatus.OK.value)
def store_feature_set( project: str, name: str, reference: str, feature_set: schemas.FeatureSet, versioned: bool = True, db_session: Session = Depends(deps.get_db_session), ): tag, uid = parse_reference(reference) uid = get_db().store_feature_set( db_session, project, name, feature_set, tag, uid, versioned ) return get_db().get_feature_set(db_session, project, name, uid=uid,)
def get_feature_set( project: str, name: str, reference: str, auth_verifier: deps.AuthVerifierDep = Depends(deps.AuthVerifierDep), db_session: Session = Depends(deps.get_db_session), ): tag, uid = parse_reference(reference) feature_set = mlrun.api.crud.FeatureStore().get_feature_set( db_session, project, name, tag, uid ) mlrun.api.utils.clients.opa.Client().query_project_resource_permissions( mlrun.api.schemas.AuthorizationResourceTypes.feature_set, project, name, mlrun.api.schemas.AuthorizationAction.read, auth_verifier.auth_info, ) return feature_set
def delete_feature_vector( project: str, name: str, reference: str = None, auth_verifier: deps.AuthVerifierDep = Depends(deps.AuthVerifierDep), db_session: Session = Depends(deps.get_db_session), ): mlrun.api.utils.clients.opa.Client().query_project_resource_permissions( mlrun.api.schemas.AuthorizationResourceTypes.feature_vector, project, name, mlrun.api.schemas.AuthorizationAction.delete, auth_verifier.auth_info, ) tag = uid = None if reference: tag, uid = parse_reference(reference) mlrun.api.crud.FeatureStore().delete_feature_vector( db_session, project, name, tag, uid ) return Response(status_code=HTTPStatus.NO_CONTENT.value)
def store_feature_set( project: str, name: str, reference: str, feature_set: schemas.FeatureSet, versioned: bool = True, auth_verifier: deps.AuthVerifier = Depends(deps.AuthVerifier), db_session: Session = Depends(deps.get_db_session), ): tag, uid = parse_reference(reference) uid = get_db().store_feature_set( db_session, project, name, feature_set, tag, uid, versioned, leader_session=auth_verifier.auth_info.session, ) return get_db().get_feature_set(db_session, project, name, tag=tag, uid=uid)
def get_feature_vector( project: str, name: str, reference: str, auth_info: mlrun.api.schemas.AuthInfo = Depends( deps.authenticate_request), db_session: Session = Depends(deps.get_db_session), ): tag, uid = parse_reference(reference) feature_vector = mlrun.api.crud.FeatureStore().get_feature_vector( db_session, project, name, tag, uid) mlrun.api.utils.auth.verifier.AuthVerifier( ).query_project_resource_permissions( mlrun.api.schemas.AuthorizationResourceTypes.feature_vector, project, name, mlrun.api.schemas.AuthorizationAction.read, auth_info, ) _verify_feature_vector_features_permissions(auth_info, project, feature_vector.dict()) return feature_vector
def ingest_feature_set( project: str, name: str, reference: str, ingest_parameters: Optional[ schemas.FeatureSetIngestInput ] = schemas.FeatureSetIngestInput(), username: str = Header(None, alias="x-remote-user"), auth_verifier: deps.AuthVerifierDep = Depends(deps.AuthVerifierDep), db_session: Session = Depends(deps.get_db_session), ): mlrun.api.utils.clients.opa.Client().query_project_resource_permissions( mlrun.api.schemas.AuthorizationResourceTypes.feature_set, project, name, mlrun.api.schemas.AuthorizationAction.update, auth_verifier.auth_info, ) mlrun.api.utils.clients.opa.Client().query_project_resource_permissions( mlrun.api.schemas.AuthorizationResourceTypes.run, project, "", mlrun.api.schemas.AuthorizationAction.create, auth_verifier.auth_info, ) data_source = data_targets = None if ingest_parameters.source: data_source = DataSource.from_dict(ingest_parameters.source.dict()) if data_source.schedule: mlrun.api.utils.clients.opa.Client().query_project_resource_permissions( mlrun.api.schemas.AuthorizationResourceTypes.schedule, project, "", mlrun.api.schemas.AuthorizationAction.create, auth_verifier.auth_info, ) tag, uid = parse_reference(reference) feature_set_record = mlrun.api.crud.FeatureStore().get_feature_set( db_session, project, name, tag, uid ) feature_set = mlrun.feature_store.FeatureSet.from_dict(feature_set_record.dict()) if feature_set.spec.function and feature_set.spec.function.function_object: function = feature_set.spec.function.function_object mlrun.api.utils.clients.opa.Client().query_project_resource_permissions( mlrun.api.schemas.AuthorizationResourceTypes.function, function.metadata.project, function.metadata.name, mlrun.api.schemas.AuthorizationAction.read, auth_verifier.auth_info, ) # Need to override the default rundb since we're in the server. feature_set._override_run_db(db_session) if ingest_parameters.targets: data_targets = [ DataTargetBase.from_dict(data_target.dict()) for data_target in ingest_parameters.targets ] run_config = RunConfig(owner=username) # Try to deduce whether the ingest job will need v3io mount, by analyzing the paths to the source and # targets. If it needs it, apply v3io mount to the run_config. Note that the access-key and username are # user-context parameters, we cannot use the api context. if _has_v3io_path(data_source, data_targets, feature_set): access_key = auth_verifier.auth_info.data_session if not access_key or not username: log_and_raise( HTTPStatus.BAD_REQUEST.value, reason="Request needs v3io access key and username in header", ) run_config = run_config.apply(v3io_cred(access_key=access_key, user=username)) infer_options = ingest_parameters.infer_options or InferOptions.default() run_params = ingest( feature_set, data_source, data_targets, infer_options=infer_options, return_df=False, run_config=run_config, ) # ingest may modify the feature-set contents, so returning the updated feature-set. result_feature_set = schemas.FeatureSet(**feature_set.to_dict()) return schemas.FeatureSetIngestOutput( feature_set=result_feature_set, run_object=run_params.to_dict() )