def store_feature_set(
project: str,
name: str,
reference: str,
feature_set: schemas.FeatureSet,
versioned: bool = True,
auth_verifier: deps.AuthVerifierDep = Depends(deps.AuthVerifierDep),
db_session: Session = Depends(deps.get_db_session),
):
mlrun.api.utils.singletons.project_member.get_project_member().ensure_project(
db_session, project, auth_info=auth_verifier.auth_info
)
mlrun.api.utils.clients.opa.Client().query_project_resource_permissions(
mlrun.api.schemas.AuthorizationResourceTypes.feature_set,
project,
name,
mlrun.api.schemas.AuthorizationAction.store,
auth_verifier.auth_info,
)
tag, uid = parse_reference(reference)
uid = mlrun.api.crud.FeatureStore().store_feature_set(
db_session, project, name, feature_set, tag, uid, versioned,
)
return mlrun.api.crud.FeatureStore().get_feature_set(
db_session, project, feature_set.metadata.name, tag, uid,
)
def patch_feature_vector(
project: str,
name: str,
feature_vector_patch: dict,
reference: str,
patch_mode: schemas.PatchMode = Header(
schemas.PatchMode.replace, alias=schemas.HeaderNames.patch_mode
),
auth_verifier: deps.AuthVerifierDep = Depends(deps.AuthVerifierDep),
db_session: Session = Depends(deps.get_db_session),
):
mlrun.api.utils.clients.opa.Client().query_project_resource_permissions(
mlrun.api.schemas.AuthorizationResourceTypes.feature_vector,
project,
name,
mlrun.api.schemas.AuthorizationAction.update,
auth_verifier.auth_info,
)
_verify_feature_vector_features_permissions(
auth_verifier.auth_info, project, feature_vector_patch
)
tag, uid = parse_reference(reference)
mlrun.api.crud.FeatureStore().patch_feature_vector(
db_session, project, name, feature_vector_patch, tag, uid, patch_mode,
)
return Response(status_code=HTTPStatus.OK.value)
def store_feature_vector(
project: str,
name: str,
reference: str,
feature_vector: schemas.FeatureVector,
versioned: bool = True,
auth_info: mlrun.api.schemas.AuthInfo = Depends(
deps.authenticate_request),
db_session: Session = Depends(deps.get_db_session),
):
mlrun.api.utils.singletons.project_member.get_project_member(
).ensure_project(db_session, project, auth_info=auth_info)
mlrun.api.utils.auth.verifier.AuthVerifier(
).query_project_resource_permissions(
mlrun.api.schemas.AuthorizationResourceTypes.feature_vector,
project,
name,
mlrun.api.schemas.AuthorizationAction.update,
auth_info,
)
_verify_feature_vector_features_permissions(auth_info, project,
feature_vector.dict())
tag, uid = parse_reference(reference)
uid = mlrun.api.crud.FeatureStore().store_feature_vector(
db_session,
project,
name,
feature_vector,
tag,
uid,
versioned,
)
return mlrun.api.crud.FeatureStore().get_feature_vector(
db_session, project, name, tag, uid)
def patch_feature_set(
project: str,
name: str,
feature_set_update: dict,
reference: str,
patch_mode: schemas.PatchMode = Header(
schemas.PatchMode.replace, alias=schemas.HeaderNames.patch_mode),
auth_info: mlrun.api.schemas.AuthInfo = Depends(
deps.authenticate_request),
db_session: Session = Depends(deps.get_db_session),
):
mlrun.api.utils.auth.verifier.AuthVerifier(
).query_project_resource_permissions(
mlrun.api.schemas.AuthorizationResourceTypes.feature_set,
project,
name,
mlrun.api.schemas.AuthorizationAction.update,
auth_info,
)
tag, uid = parse_reference(reference)
mlrun.api.crud.FeatureStore().patch_feature_set(
db_session,
project,
name,
feature_set_update,
tag,
uid,
patch_mode,
)
return Response(status_code=HTTPStatus.OK.value)
def ingest_feature_set(
request: Request,
project: str,
name: str,
reference: str,
ingest_parameters: Optional[
schemas.FeatureSetIngestInput
] = schemas.FeatureSetIngestInput(),
username: str = Header(None, alias="x-remote-user"),
auth_verifier: deps.AuthVerifier = Depends(deps.AuthVerifier),
db_session: Session = Depends(deps.get_db_session),
):
tag, uid = parse_reference(reference)
feature_set_record = get_db().get_feature_set(db_session, project, name, tag, uid)
feature_set = mlrun.feature_store.FeatureSet.from_dict(feature_set_record.dict())
# Need to override the default rundb since we're in the server.
feature_set._override_run_db(db_session, auth_verifier.auth_info.session)
data_source = data_targets = None
if ingest_parameters.source:
data_source = DataSource.from_dict(ingest_parameters.source.dict())
if ingest_parameters.targets:
data_targets = [
DataTargetBase.from_dict(data_target.dict())
for data_target in ingest_parameters.targets
]
run_config = RunConfig()
# Try to deduce whether the ingest job will need v3io mount, by analyzing the paths to the source and
# targets. If it needs it, apply v3io mount to the run_config. Note that the access-key and username are
# user-context parameters, we cannot use the api context.
if _has_v3io_path(data_source, data_targets, feature_set):
secrets = get_secrets(request)
access_key = secrets.get("V3IO_ACCESS_KEY", None)
if not access_key or not username:
log_and_raise(
HTTPStatus.BAD_REQUEST.value,
reason="Request needs v3io access key and username in header",
)
run_config = run_config.apply(v3io_cred(access_key=access_key, user=username))
infer_options = ingest_parameters.infer_options or InferOptions.default()
run_params = ingest(
feature_set,
data_source,
data_targets,
infer_options=infer_options,
return_df=False,
run_config=run_config,
)
# ingest may modify the feature-set contents, so returning the updated feature-set.
result_feature_set = schemas.FeatureSet(**feature_set.to_dict())
return schemas.FeatureSetIngestOutput(
feature_set=result_feature_set, run_object=run_params.to_dict()
)
def get_feature_vector(
project: str,
name: str,
reference: str,
db_session: Session = Depends(deps.get_db_session),
):
tag, uid = parse_reference(reference)
return get_db().get_feature_vector(db_session, project, name, tag, uid)
def update_feature_set(
project: str,
name: str,
feature_set_update: schemas.FeatureSetUpdate,
reference: str,
db_session: Session = Depends(deps.get_db_session),
):
tag, uid = parse_reference(reference)
get_db().update_feature_set(db_session, project, name, feature_set_update,
tag, uid)
return Response(status_code=HTTPStatus.OK.value)
def delete_feature_set(
project: str,
name: str,
reference: str = None,
db_session: Session = Depends(deps.get_db_session),
):
tag = uid = None
if reference:
tag, uid = parse_reference(reference)
get_db().delete_feature_set(db_session, project, name, tag, uid)
return Response(status_code=HTTPStatus.NO_CONTENT.value)
def patch_feature_set(
project: str,
name: str,
feature_set_update: dict,
reference: str,
patch_mode: schemas.PatchMode = Header(
schemas.PatchMode.replace, alias=schemas.HeaderNames.patch_mode),
db_session: Session = Depends(deps.get_db_session),
):
tag, uid = parse_reference(reference)
get_db().patch_feature_set(db_session, project, name, feature_set_update,
tag, uid, patch_mode)
return Response(status_code=HTTPStatus.OK.value)
def store_feature_set(
project: str,
name: str,
reference: str,
feature_set: schemas.FeatureSet,
versioned: bool = True,
db_session: Session = Depends(deps.get_db_session),
):
tag, uid = parse_reference(reference)
uid = get_db().store_feature_set(
db_session, project, name, feature_set, tag, uid, versioned
)
return get_db().get_feature_set(db_session, project, name, uid=uid,)
def get_feature_set(
project: str,
name: str,
reference: str,
auth_verifier: deps.AuthVerifierDep = Depends(deps.AuthVerifierDep),
db_session: Session = Depends(deps.get_db_session),
):
tag, uid = parse_reference(reference)
feature_set = mlrun.api.crud.FeatureStore().get_feature_set(
db_session, project, name, tag, uid
)
mlrun.api.utils.clients.opa.Client().query_project_resource_permissions(
mlrun.api.schemas.AuthorizationResourceTypes.feature_set,
project,
name,
mlrun.api.schemas.AuthorizationAction.read,
auth_verifier.auth_info,
)
return feature_set
def delete_feature_vector(
project: str,
name: str,
reference: str = None,
auth_verifier: deps.AuthVerifierDep = Depends(deps.AuthVerifierDep),
db_session: Session = Depends(deps.get_db_session),
):
mlrun.api.utils.clients.opa.Client().query_project_resource_permissions(
mlrun.api.schemas.AuthorizationResourceTypes.feature_vector,
project,
name,
mlrun.api.schemas.AuthorizationAction.delete,
auth_verifier.auth_info,
)
tag = uid = None
if reference:
tag, uid = parse_reference(reference)
mlrun.api.crud.FeatureStore().delete_feature_vector(
db_session, project, name, tag, uid
)
return Response(status_code=HTTPStatus.NO_CONTENT.value)
def store_feature_set(
project: str,
name: str,
reference: str,
feature_set: schemas.FeatureSet,
versioned: bool = True,
auth_verifier: deps.AuthVerifier = Depends(deps.AuthVerifier),
db_session: Session = Depends(deps.get_db_session),
):
tag, uid = parse_reference(reference)
uid = get_db().store_feature_set(
db_session,
project,
name,
feature_set,
tag,
uid,
versioned,
leader_session=auth_verifier.auth_info.session,
)
return get_db().get_feature_set(db_session, project, name, tag=tag, uid=uid)
def get_feature_vector(
project: str,
name: str,
reference: str,
auth_info: mlrun.api.schemas.AuthInfo = Depends(
deps.authenticate_request),
db_session: Session = Depends(deps.get_db_session),
):
tag, uid = parse_reference(reference)
feature_vector = mlrun.api.crud.FeatureStore().get_feature_vector(
db_session, project, name, tag, uid)
mlrun.api.utils.auth.verifier.AuthVerifier(
).query_project_resource_permissions(
mlrun.api.schemas.AuthorizationResourceTypes.feature_vector,
project,
name,
mlrun.api.schemas.AuthorizationAction.read,
auth_info,
)
_verify_feature_vector_features_permissions(auth_info, project,
feature_vector.dict())
return feature_vector
def ingest_feature_set(
project: str,
name: str,
reference: str,
ingest_parameters: Optional[
schemas.FeatureSetIngestInput
] = schemas.FeatureSetIngestInput(),
username: str = Header(None, alias="x-remote-user"),
auth_verifier: deps.AuthVerifierDep = Depends(deps.AuthVerifierDep),
db_session: Session = Depends(deps.get_db_session),
):
mlrun.api.utils.clients.opa.Client().query_project_resource_permissions(
mlrun.api.schemas.AuthorizationResourceTypes.feature_set,
project,
name,
mlrun.api.schemas.AuthorizationAction.update,
auth_verifier.auth_info,
)
mlrun.api.utils.clients.opa.Client().query_project_resource_permissions(
mlrun.api.schemas.AuthorizationResourceTypes.run,
project,
"",
mlrun.api.schemas.AuthorizationAction.create,
auth_verifier.auth_info,
)
data_source = data_targets = None
if ingest_parameters.source:
data_source = DataSource.from_dict(ingest_parameters.source.dict())
if data_source.schedule:
mlrun.api.utils.clients.opa.Client().query_project_resource_permissions(
mlrun.api.schemas.AuthorizationResourceTypes.schedule,
project,
"",
mlrun.api.schemas.AuthorizationAction.create,
auth_verifier.auth_info,
)
tag, uid = parse_reference(reference)
feature_set_record = mlrun.api.crud.FeatureStore().get_feature_set(
db_session, project, name, tag, uid
)
feature_set = mlrun.feature_store.FeatureSet.from_dict(feature_set_record.dict())
if feature_set.spec.function and feature_set.spec.function.function_object:
function = feature_set.spec.function.function_object
mlrun.api.utils.clients.opa.Client().query_project_resource_permissions(
mlrun.api.schemas.AuthorizationResourceTypes.function,
function.metadata.project,
function.metadata.name,
mlrun.api.schemas.AuthorizationAction.read,
auth_verifier.auth_info,
)
# Need to override the default rundb since we're in the server.
feature_set._override_run_db(db_session)
if ingest_parameters.targets:
data_targets = [
DataTargetBase.from_dict(data_target.dict())
for data_target in ingest_parameters.targets
]
run_config = RunConfig(owner=username)
# Try to deduce whether the ingest job will need v3io mount, by analyzing the paths to the source and
# targets. If it needs it, apply v3io mount to the run_config. Note that the access-key and username are
# user-context parameters, we cannot use the api context.
if _has_v3io_path(data_source, data_targets, feature_set):
access_key = auth_verifier.auth_info.data_session
if not access_key or not username:
log_and_raise(
HTTPStatus.BAD_REQUEST.value,
reason="Request needs v3io access key and username in header",
)
run_config = run_config.apply(v3io_cred(access_key=access_key, user=username))
infer_options = ingest_parameters.infer_options or InferOptions.default()
run_params = ingest(
feature_set,
data_source,
data_targets,
infer_options=infer_options,
return_df=False,
run_config=run_config,
)
# ingest may modify the feature-set contents, so returning the updated feature-set.
result_feature_set = schemas.FeatureSet(**feature_set.to_dict())
return schemas.FeatureSetIngestOutput(
feature_set=result_feature_set, run_object=run_params.to_dict()
)
