async def create_download_session(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict({
t.Key('volume'): t.String(),
t.Key('vfid'): tx.UUID(),
t.Key('relpath'): tx.PurePath(relative_only=True),
t.Key('archive', default=False): t.ToBool,
t.Key('unmanaged_path', default=None): t.Null | t.String,
})) as params:
await log_manager_api_entry(log, 'create_download_session', params)
ctx: Context = request.app['ctx']
token_data = {
'op':
'download',
'volume':
params['volume'],
'vfid':
str(params['vfid']),
'relpath':
str(params['relpath']),
'exp':
datetime.utcnow() +
ctx.local_config['storage-proxy']['session-expire'],
}
token = jwt.encode(
token_data,
ctx.local_config['storage-proxy']['secret'],
algorithm='HS256',
).decode('UTF-8')
return web.json_response({
'token': token,
})
async def create_download_session(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict(
{
t.Key("volume"): t.String(),
t.Key("vfid"): tx.UUID(),
t.Key("relpath"): tx.PurePath(relative_only=True),
t.Key("archive", default=False): t.ToBool,
t.Key("unmanaged_path", default=None): t.Null | t.String,
}, ),
) as params:
await log_manager_api_entry(log, "create_download_session", params)
ctx: Context = request.app["ctx"]
token_data = {
"op":
"download",
"volume":
params["volume"],
"vfid":
str(params["vfid"]),
"relpath":
str(params["relpath"]),
"exp":
datetime.utcnow() +
ctx.local_config["storage-proxy"]["session-expire"],
}
token = jwt.encode(
token_data,
ctx.local_config["storage-proxy"]["secret"],
algorithm="HS256",
)
return web.json_response({
"token": token,
}, )
async def list_files(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict({
t.Key('volume'): t.String(),
t.Key('vfid'): tx.UUID(),
t.Key('relpath'): tx.PurePath(relative_only=True),
})) as params:
await log_manager_api_entry(log, 'list_files', params)
ctx: Context = request.app['ctx']
async with ctx.get_volume(params['volume']) as volume:
items = [{
'name': item.name,
'type': item.type.name,
'stat': {
'mode': item.stat.mode,
'size': item.stat.size,
'created': item.stat.created.isoformat(),
'modified': item.stat.modified.isoformat(),
},
} async for item in volume.scandir(
params['vfid'],
params['relpath'],
)]
return web.json_response({
'items': items,
})
async def list_files(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict(
{
t.Key("volume"): t.String(),
t.Key("vfid"): tx.UUID(),
t.Key("relpath"): tx.PurePath(relative_only=True),
}, ),
) as params:
await log_manager_api_entry(log, "list_files", params)
ctx: Context = request.app["ctx"]
async with ctx.get_volume(params["volume"]) as volume:
with handle_fs_errors(volume, params["vfid"]):
items = [{
"name": item.name,
"type": item.type.name,
"stat": {
"mode": item.stat.mode,
"size": item.stat.size,
"created": item.stat.created.isoformat(),
"modified": item.stat.modified.isoformat(),
},
"symlink_target": item.symlink_target,
} async for item in volume.scandir(
params["vfid"],
params["relpath"],
)]
return web.json_response({
"items": items,
}, )
def as_trafaret(cls) -> t.Trafaret:
return t.Dict(
{
t.Key("backend"): t.String,
t.Key("path"): tx.Path(type="dir"),
t.Key("fsprefix", default="."): tx.PurePath(relative_only=True),
t.Key("options", default=None): t.Null | t.Mapping(t.String, t.Any),
},
)
def as_trafaret(cls) -> t.Trafaret:
return t.Dict({
t.Key('backend'):
t.String,
t.Key('path'):
tx.Path(type='dir'),
t.Key('fsprefix', default='.'):
tx.PurePath(relative_only=True),
t.Key('options', default=None):
t.Null | t.Mapping(t.String, t.Any),
})
async def delete_files(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict({
t.Key('volume'): t.String(),
t.Key('vfid'): tx.UUID(),
t.Key('relpaths'): t.List(tx.PurePath(relative_only=True)),
})) as params:
await log_manager_api_entry(log, 'delete_files', params)
return web.json_response({
'status': 'ok',
})
async def move_file(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict(
{
t.Key("volume"): t.String(),
t.Key("vfid"): tx.UUID(),
t.Key("src_relpath"): tx.PurePath(relative_only=True),
t.Key("dst_relpath"): tx.PurePath(relative_only=True),
}, ),
) as params:
await log_manager_api_entry(log, "move_file", params)
ctx: Context = request.app["ctx"]
async with ctx.get_volume(params["volume"]) as volume:
with handle_fs_errors(volume, params["vfid"]):
await volume.move_file(
params["vfid"],
params["src_relpath"],
params["dst_relpath"],
)
return web.Response(status=204)
async def mkdir(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict({
t.Key('volume'): t.String(),
t.Key('vfid'): tx.UUID(),
t.Key('relpath'): tx.PurePath(relative_only=True),
})) as params:
await log_manager_api_entry(log, 'mkdir', params)
ctx: Context = request.app['ctx']
async with ctx.get_volume(params['volume']) as volume:
await volume.mkdir(params['vfid'], params['relpath'], parents=True)
return web.Response(status=204)
async def create_upload_session(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict(
{
t.Key("volume"): t.String(),
t.Key("vfid"): tx.UUID(),
t.Key("relpath"): tx.PurePath(relative_only=True),
t.Key("size"): t.ToInt,
}, ),
) as params:
await log_manager_api_entry(log, "create_upload_session", params)
ctx: Context = request.app["ctx"]
async with ctx.get_volume(params["volume"]) as volume:
session_id = await volume.prepare_upload(params["vfid"])
token_data = {
"op":
"upload",
"volume":
params["volume"],
"vfid":
str(params["vfid"]),
"relpath":
str(params["relpath"]),
"size":
params["size"],
"session":
session_id,
"exp":
datetime.utcnow() +
ctx.local_config["storage-proxy"]["session-expire"],
}
token = jwt.encode(
token_data,
ctx.local_config["storage-proxy"]["secret"],
algorithm="HS256",
)
return web.json_response({
"token": token,
}, )
async def rename_file(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict(
{
t.Key("volume"): t.String(),
t.Key("vfid"): tx.UUID(),
t.Key("relpath"): tx.PurePath(relative_only=True),
t.Key("new_name"): t.String(),
t.Key("is_dir"): t.ToBool(), # ignored since 22.03
}, ),
) as params:
await log_manager_api_entry(log, "rename_file", params)
ctx: Context = request.app["ctx"]
async with ctx.get_volume(params["volume"]) as volume:
with handle_fs_errors(volume, params["vfid"]):
await volume.move_file(
params["vfid"],
params["relpath"],
params["relpath"].with_name(params["new_name"]),
)
return web.Response(status=204)
async def create_upload_session(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict({
t.Key('volume'): t.String(),
t.Key('vfid'): tx.UUID(),
t.Key('relpath'): tx.PurePath(relative_only=True),
t.Key('size'): t.ToInt,
})) as params:
await log_manager_api_entry(log, 'create_upload_session', params)
ctx: Context = request.app['ctx']
async with ctx.get_volume(params['volume']) as volume:
session_id = await volume.prepare_upload(params['vfid'])
token_data = {
'op':
'upload',
'volume':
params['volume'],
'vfid':
str(params['vfid']),
'relpath':
str(params['relpath']),
'size':
params['size'],
'session':
session_id,
'exp':
datetime.utcnow() +
ctx.local_config['storage-proxy']['session-expire'],
}
token = jwt.encode(
token_data,
ctx.local_config['storage-proxy']['secret'],
algorithm='HS256',
).decode('UTF-8')
return web.json_response({
'token': token,
})
async def fetch_file(request: web.Request) -> web.StreamResponse:
"""
Direct file streaming API for internal use, such as retrieving
task logs from a user vfolder ".logs".
"""
async with check_params(
request,
t.Dict(
{
t.Key("volume"): t.String(),
t.Key("vfid"): tx.UUID(),
t.Key("relpath"): tx.PurePath(relative_only=True),
}, ),
) as params:
await log_manager_api_entry(log, "fetch_file", params)
ctx: Context = request.app["ctx"]
response = web.StreamResponse(status=200)
response.headers[hdrs.CONTENT_TYPE] = "application/octet-stream"
try:
prepared = False
async with ctx.get_volume(params["volume"]) as volume:
with handle_fs_errors(volume, params["vfid"]):
async for chunk in volume.read_file(
params["vfid"],
params["relpath"],
):
if not chunk:
return response
if not prepared:
await response.prepare(request)
prepared = True
await response.write(chunk)
except FileNotFoundError:
response = web.Response(status=404, reason="Log data not found")
finally:
if prepared:
await response.write_eof()
return response
async def delete_files(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict(
{
t.Key("volume"): t.String(),
t.Key("vfid"): tx.UUID(),
t.Key("relpaths"): t.List(tx.PurePath(relative_only=True)),
t.Key("recursive", default=False): t.ToBool,
}, ),
) as params:
await log_manager_api_entry(log, "delete_files", params)
ctx: Context = request.app["ctx"]
async with ctx.get_volume(params["volume"]) as volume:
with handle_fs_errors(volume, params["vfid"]):
await volume.delete_files(
params["vfid"],
params["relpaths"],
params["recursive"],
)
return web.json_response({
"status": "ok",
}, )
async def mkdir(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict(
{
t.Key("volume"): t.String(),
t.Key("vfid"): tx.UUID(),
t.Key("relpath"): tx.PurePath(relative_only=True),
t.Key("parents", default=True): t.ToBool,
t.Key("exist_ok", default=False): t.ToBool,
}, ),
) as params:
await log_manager_api_entry(log, "mkdir", params)
ctx: Context = request.app["ctx"]
async with ctx.get_volume(params["volume"]) as volume:
with handle_fs_errors(volume, params["vfid"]):
await volume.mkdir(
params["vfid"],
params["relpath"],
parents=params["parents"],
exist_ok=params["exist_ok"],
)
return web.Response(status=204)