async def create_vfolder(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict(
{
t.Key("volume"):
t.String(),
t.Key("vfid"):
tx.UUID(),
t.Key("options", default=None):
t.Null
| VFolderCreationOptions.as_trafaret(),
}, ),
) as params:
await log_manager_api_entry(log, "create_vfolder", params)
ctx: Context = request.app["ctx"]
async with ctx.get_volume(params["volume"]) as volume:
obj_opts = VFolderCreationOptions.as_object(params["options"])
await volume.create_vfolder(params["vfid"], obj_opts)
return web.Response(status=204)
async def move_file(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict(
{
t.Key("volume"): t.String(),
t.Key("vfid"): tx.UUID(),
t.Key("src_relpath"): tx.PurePath(relative_only=True),
t.Key("dst_relpath"): tx.PurePath(relative_only=True),
}, ),
) as params:
await log_manager_api_entry(log, "move_file", params)
ctx: Context = request.app["ctx"]
async with ctx.get_volume(params["volume"]) as volume:
with handle_fs_errors(volume, params["vfid"]):
await volume.move_file(
params["vfid"],
params["src_relpath"],
params["dst_relpath"],
)
return web.Response(status=204)
async def create_upload_session(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict(
{
t.Key("volume"): t.String(),
t.Key("vfid"): tx.UUID(),
t.Key("relpath"): tx.PurePath(relative_only=True),
t.Key("size"): t.ToInt,
}, ),
) as params:
await log_manager_api_entry(log, "create_upload_session", params)
ctx: Context = request.app["ctx"]
async with ctx.get_volume(params["volume"]) as volume:
session_id = await volume.prepare_upload(params["vfid"])
token_data = {
"op":
"upload",
"volume":
params["volume"],
"vfid":
str(params["vfid"]),
"relpath":
str(params["relpath"]),
"size":
params["size"],
"session":
session_id,
"exp":
datetime.utcnow() +
ctx.local_config["storage-proxy"]["session-expire"],
}
token = jwt.encode(
token_data,
ctx.local_config["storage-proxy"]["secret"],
algorithm="HS256",
)
return web.json_response({
"token": token,
}, )
async def get_vfolder_mount(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict(
{
t.Key("volume"): t.String(),
t.Key("vfid"): tx.UUID(),
t.Key("subpath", default="."): t.String(),
}, ),
) as params:
await log_manager_api_entry(log, "get_container_mount", params)
ctx: Context = request.app["ctx"]
async with ctx.get_volume(params["volume"]) as volume:
try:
mount_path = await volume.get_vfolder_mount(
params["vfid"],
params["subpath"],
)
except VFolderNotFoundError:
raise web.HTTPBadRequest(
body=json.dumps(
{
"msg": "VFolder not found",
"vfid": str(params["vfid"]),
}, ),
content_type="application/json",
)
except InvalidSubpathError as e:
raise web.HTTPBadRequest(
body=json.dumps(
{
"msg": "Invalid vfolder subpath",
"vfid": str(params["vfid"]),
"subpath": str(e.args[1]),
}, ),
content_type="application/json",
)
return web.json_response({
"path": str(mount_path),
}, )
async def rename_file(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict(
{
t.Key("volume"): t.String(),
t.Key("vfid"): tx.UUID(),
t.Key("relpath"): tx.PurePath(relative_only=True),
t.Key("new_name"): t.String(),
t.Key("is_dir"): t.ToBool(), # ignored since 22.03
}, ),
) as params:
await log_manager_api_entry(log, "rename_file", params)
ctx: Context = request.app["ctx"]
async with ctx.get_volume(params["volume"]) as volume:
with handle_fs_errors(volume, params["vfid"]):
await volume.move_file(
params["vfid"],
params["relpath"],
params["relpath"].with_name(params["new_name"]),
)
return web.Response(status=204)
async def create_upload_session(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict({
t.Key('volume'): t.String(),
t.Key('vfid'): tx.UUID(),
t.Key('relpath'): tx.PurePath(relative_only=True),
t.Key('size'): t.ToInt,
})) as params:
await log_manager_api_entry(log, 'create_upload_session', params)
ctx: Context = request.app['ctx']
async with ctx.get_volume(params['volume']) as volume:
session_id = await volume.prepare_upload(params['vfid'])
token_data = {
'op':
'upload',
'volume':
params['volume'],
'vfid':
str(params['vfid']),
'relpath':
str(params['relpath']),
'size':
params['size'],
'session':
session_id,
'exp':
datetime.utcnow() +
ctx.local_config['storage-proxy']['session-expire'],
}
token = jwt.encode(
token_data,
ctx.local_config['storage-proxy']['secret'],
algorithm='HS256',
).decode('UTF-8')
return web.json_response({
'token': token,
})
async def fetch_file(request: web.Request) -> web.StreamResponse:
"""
Direct file streaming API for internal use, such as retrieving
task logs from a user vfolder ".logs".
"""
async with check_params(
request,
t.Dict(
{
t.Key("volume"): t.String(),
t.Key("vfid"): tx.UUID(),
t.Key("relpath"): tx.PurePath(relative_only=True),
}, ),
) as params:
await log_manager_api_entry(log, "fetch_file", params)
ctx: Context = request.app["ctx"]
response = web.StreamResponse(status=200)
response.headers[hdrs.CONTENT_TYPE] = "application/octet-stream"
try:
prepared = False
async with ctx.get_volume(params["volume"]) as volume:
with handle_fs_errors(volume, params["vfid"]):
async for chunk in volume.read_file(
params["vfid"],
params["relpath"],
):
if not chunk:
return response
if not prepared:
await response.prepare(request)
prepared = True
await response.write(chunk)
except FileNotFoundError:
response = web.Response(status=404, reason="Log data not found")
finally:
if prepared:
await response.write_eof()
return response
async def delete_files(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict(
{
t.Key("volume"): t.String(),
t.Key("vfid"): tx.UUID(),
t.Key("relpaths"): t.List(tx.PurePath(relative_only=True)),
t.Key("recursive", default=False): t.ToBool,
}, ),
) as params:
await log_manager_api_entry(log, "delete_files", params)
ctx: Context = request.app["ctx"]
async with ctx.get_volume(params["volume"]) as volume:
with handle_fs_errors(volume, params["vfid"]):
await volume.delete_files(
params["vfid"],
params["relpaths"],
params["recursive"],
)
return web.json_response({
"status": "ok",
}, )
async def mkdir(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict(
{
t.Key("volume"): t.String(),
t.Key("vfid"): tx.UUID(),
t.Key("relpath"): tx.PurePath(relative_only=True),
t.Key("parents", default=True): t.ToBool,
t.Key("exist_ok", default=False): t.ToBool,
}, ),
) as params:
await log_manager_api_entry(log, "mkdir", params)
ctx: Context = request.app["ctx"]
async with ctx.get_volume(params["volume"]) as volume:
with handle_fs_errors(volume, params["vfid"]):
await volume.mkdir(
params["vfid"],
params["relpath"],
parents=params["parents"],
exist_ok=params["exist_ok"],
)
return web.Response(status=204)
async def get_vfolder_usage(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict({
t.Key("volume"): t.String(),
t.Key("vfid"): tx.UUID(),
}, ),
) as params:
try:
await log_manager_api_entry(log, "get_vfolder_usage", params)
ctx: Context = request.app["ctx"]
async with ctx.get_volume(params["volume"]) as volume:
usage = await volume.get_usage(params["vfid"])
return web.json_response(
{
"file_count": usage.file_count,
"used_bytes": usage.used_bytes,
}, )
except ExecutionError:
return web.Response(
status=500,
reason="Storage server is busy. Please try again",
)