def get_filestat(request: Request, schema: str = "", path: str = "", user: str = ""):
_, filename = path.split(path)
path = get_obj_path(schema, path, user=user)
if not path:
log_and_raise(
HTTPStatus.NOT_FOUND.value, path=path, err="illegal path prefix or schema"
)
logger.debug("Got get filestat request", path=path)
secrets = get_secrets(request)
stat = None
try:
stores = store_manager.set(secrets)
stat = stores.object(url=path).stat()
except FileNotFoundError as exc:
log_and_raise(HTTPStatus.NOT_FOUND.value, path=path, err=str(exc))
ctype, _ = mimetypes.guess_type(path)
if not ctype:
ctype = "application/octet-stream"
return {
"size": stat.size,
"modified": stat.modified,
"mimetype": ctype,
}
def get_filestat(request: Request,
schema: str = "",
path: str = "",
user: str = ""):
_, filename = path.split(path)
path = get_obj_path(schema, path, user=user)
if not path:
log_and_raise(HTTPStatus.NOT_FOUND,
path=path,
err="illegal path prefix or schema")
secrets = get_secrets(request)
stat = None
try:
stat = get_object_stat(path, secrets)
except FileNotFoundError as e:
log_and_raise(HTTPStatus.NOT_FOUND, path=path, err=str(e))
ctype, _ = mimetypes.guess_type(path)
if not ctype:
ctype = "application/octet-stream"
return {
"size": stat.size,
"modified": stat.modified,
"mimetype": ctype,
}
def ingest_feature_set(
request: Request,
project: str,
name: str,
reference: str,
ingest_parameters: Optional[
schemas.FeatureSetIngestInput
] = schemas.FeatureSetIngestInput(),
username: str = Header(None, alias="x-remote-user"),
auth_verifier: deps.AuthVerifier = Depends(deps.AuthVerifier),
db_session: Session = Depends(deps.get_db_session),
):
tag, uid = parse_reference(reference)
feature_set_record = get_db().get_feature_set(db_session, project, name, tag, uid)
feature_set = mlrun.feature_store.FeatureSet.from_dict(feature_set_record.dict())
# Need to override the default rundb since we're in the server.
feature_set._override_run_db(db_session, auth_verifier.auth_info.session)
data_source = data_targets = None
if ingest_parameters.source:
data_source = DataSource.from_dict(ingest_parameters.source.dict())
if ingest_parameters.targets:
data_targets = [
DataTargetBase.from_dict(data_target.dict())
for data_target in ingest_parameters.targets
]
run_config = RunConfig()
# Try to deduce whether the ingest job will need v3io mount, by analyzing the paths to the source and
# targets. If it needs it, apply v3io mount to the run_config. Note that the access-key and username are
# user-context parameters, we cannot use the api context.
if _has_v3io_path(data_source, data_targets, feature_set):
secrets = get_secrets(request)
access_key = secrets.get("V3IO_ACCESS_KEY", None)
if not access_key or not username:
log_and_raise(
HTTPStatus.BAD_REQUEST.value,
reason="Request needs v3io access key and username in header",
)
run_config = run_config.apply(v3io_cred(access_key=access_key, user=username))
infer_options = ingest_parameters.infer_options or InferOptions.default()
run_params = ingest(
feature_set,
data_source,
data_targets,
infer_options=infer_options,
return_df=False,
run_config=run_config,
)
# ingest may modify the feature-set contents, so returning the updated feature-set.
result_feature_set = schemas.FeatureSet(**feature_set.to_dict())
return schemas.FeatureSetIngestOutput(
feature_set=result_feature_set, run_object=run_params.to_dict()
)
def get_files(
schema: str = "",
objpath: str = fastapi.Query("", alias="path"),
user: str = "",
size: int = 0,
offset: int = 0,
auth_info: mlrun.api.schemas.AuthInfo = fastapi.Depends(
mlrun.api.api.deps.authenticate_request),
):
_, filename = objpath.split(objpath)
objpath = get_obj_path(schema, objpath, user=user)
if not objpath:
log_and_raise(
HTTPStatus.NOT_FOUND.value,
path=objpath,
err="illegal path prefix or schema",
)
logger.debug("Got get files request", path=objpath)
secrets = get_secrets(auth_info)
body = None
try:
stores = store_manager.set(secrets)
obj = stores.object(url=objpath)
if objpath.endswith("/"):
listdir = obj.listdir()
return {
"listdir": listdir,
}
body = obj.get(size, offset)
except FileNotFoundError as exc:
log_and_raise(HTTPStatus.NOT_FOUND.value, path=objpath, err=str(exc))
if body is None:
log_and_raise(HTTPStatus.NOT_FOUND.value, path=objpath)
ctype, _ = mimetypes.guess_type(objpath)
if not ctype:
ctype = "application/octet-stream"
return fastapi.Response(content=body,
media_type=ctype,
headers={"x-suggested-filename": filename})
def get_files(
request: Request,
schema: str = "",
objpath: str = Query("", alias="path"),
user: str = "",
size: int = 0,
offset: int = 0,
):
_, filename = objpath.split(objpath)
objpath = get_obj_path(schema, objpath, user=user)
if not objpath:
log_and_raise(
HTTPStatus.NOT_FOUND, path=objpath, err="illegal path prefix or schema"
)
secrets = get_secrets(request)
body = None
try:
stores = StoreManager(secrets)
obj = stores.object(url=objpath)
if objpath.endswith("/"):
listdir = obj.listdir()
return {
"listdir": listdir,
}
body = obj.get(size, offset)
except FileNotFoundError as e:
log_and_raise(HTTPStatus.NOT_FOUND, path=objpath, err=str(e))
if body is None:
log_and_raise(HTTPStatus.NOT_FOUND, path=objpath)
ctype, _ = mimetypes.guess_type(objpath)
if not ctype:
ctype = "application/octet-stream"
return Response(
content=body, media_type=ctype, headers={"x-suggested-filename": filename}
)