async def create_vfolder(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict(
{
t.Key("volume"):
t.String(),
t.Key("vfid"):
tx.UUID(),
t.Key("options", default=None):
t.Null
| VFolderCreationOptions.as_trafaret(),
}, ),
) as params:
await log_manager_api_entry(log, "create_vfolder", params)
ctx: Context = request.app["ctx"]
async with ctx.get_volume(params["volume"]) as volume:
obj_opts = VFolderCreationOptions.as_object(params["options"])
await volume.create_vfolder(params["vfid"], obj_opts)
return web.Response(status=204)
class BatchInfoTrafaret(trf.Trafaret):
scheme = trf.Dict(
{
trf.Key("id", optional=True): trf.String,
trf.Key("name", optional=True): trf.String,
trf.Key("batch_sequence_name", optional=True)
>> "sequence_name": trf.String,
trf.Key("started_at", optional=True): trf.DateTime,
trf.Key("properties", optional=True): trf.List(
trf.Dict(name=trf.String, value=trf.String)
),
},
)
def check_and_return(self, value, context=None):
sanitized = self.scheme.check(value, context=context)
batch = BatchInfo()
for key, val in sanitized.items():
setattr(batch, key, val)
return batch
class FlatPage(EmbeddedDocument):
""" A flatpage representation model
"""
structure = t.Dict({
'title': t.String,
'slug': t.String,
'content': t.String,
'login_required': t.Bool,
t.Key('template', default=''): t.String,
})
required_fields = ['title', 'slug', 'content']
def test_callable_key(self):
def simple_key(value):
yield 'simple', 'simple data', []
trafaret = t.Dict(simple_key)
res = trafaret.check({})
assert res == {'simple': 'simple data'}
trafaret = t.Dict({t.Key('key'): t.String}, simple_key)
res = trafaret.check({'key': u'blabla'})
assert res == {'key': u'blabla', 'simple': 'simple data'}
async def get_performance_metric(request: web.Request) -> web.Response:
async with check_params(request, t.Dict({
t.Key('volume'): t.String(),
})) as params:
await log_manager_api_entry(log, 'get_performance_metric', params)
ctx: Context = request.app['ctx']
async with ctx.get_volume(params['volume']) as volume:
metric = await volume.get_performance_metric()
return web.json_response({
'metric': attr.asdict(metric),
})
def test_add_kwargs_ignore(self):
first = t.Dict(
t.Key('bar', trafaret=t.Int()), ignore_extra=['eggs']
)
second = t.Dict(
t.Key('bar1', trafaret=t.Int())
)
third = first + second
third.check({'bar': 4, 'bar1': 41})
third.check({'bar': 4, 'bar1': 41, 'eggs': 'blabla'})
first = t.Dict(
t.Key('bar', trafaret=t.Int()),
)
second = t.Dict(
t.Key('bar1', trafaret=t.Int()), ignore_extra=['eggs']
)
third = first + second
third.check({'bar': 4, 'bar1': 41})
third.check({'bar': 4, 'bar1': 41, 'eggs': 'blabla'})
async def get_hwinfo(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict({
t.Key("volume"): t.String(),
}, ),
) as params:
await log_manager_api_entry(log, "get_hwinfo", params)
ctx: Context = request.app["ctx"]
async with ctx.get_volume(params["volume"]) as volume:
data = await volume.get_hwinfo()
return web.json_response(data)
def test_2_0_regression(self):
t_request = t.Dict({
t.Key('params', optional=True):
t.Or(t.List(t.Any()), t.Mapping(t.AnyString(), t.Any())),
})
assert t_request.check({'params': {
'aaa': 123
}}) == {
'params': {
'aaa': 123
}
}
def test_args_checks(self):
with pytest.raises(RuntimeError) as exc_info:
@guard(123)
def fn(**kw):
return kw
assert exc_info.value.args[0] == 'trafaret should be instance of Dict or Forward'
with pytest.raises(RuntimeError) as exc_info:
@guard(t.Dict(t.Key('a', trafaret=t.Bytes)), a=t.ToInt)
def fn(**kw):
return kw
assert exc_info.value.args[0] == 'choose one way of initialization, trafaret or kwargs'
async def move_file(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict(
{
t.Key("volume"): t.String(),
t.Key("vfid"): tx.UUID(),
t.Key("src_relpath"): tx.PurePath(relative_only=True),
t.Key("dst_relpath"): tx.PurePath(relative_only=True),
}, ),
) as params:
await log_manager_api_entry(log, "move_file", params)
ctx: Context = request.app["ctx"]
async with ctx.get_volume(params["volume"]) as volume:
with handle_fs_errors(volume, params["vfid"]):
await volume.move_file(
params["vfid"],
params["src_relpath"],
params["dst_relpath"],
)
return web.Response(status=204)
async def create_upload_session(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict(
{
t.Key("volume"): t.String(),
t.Key("vfid"): tx.UUID(),
t.Key("relpath"): tx.PurePath(relative_only=True),
t.Key("size"): t.ToInt,
}, ),
) as params:
await log_manager_api_entry(log, "create_upload_session", params)
ctx: Context = request.app["ctx"]
async with ctx.get_volume(params["volume"]) as volume:
session_id = await volume.prepare_upload(params["vfid"])
token_data = {
"op":
"upload",
"volume":
params["volume"],
"vfid":
str(params["vfid"]),
"relpath":
str(params["relpath"]),
"size":
params["size"],
"session":
session_id,
"exp":
datetime.utcnow() +
ctx.local_config["storage-proxy"]["session-expire"],
}
token = jwt.encode(
token_data,
ctx.local_config["storage-proxy"]["secret"],
algorithm="HS256",
)
return web.json_response({
"token": token,
}, )
async def get_vfolder_mount(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict(
{
t.Key("volume"): t.String(),
t.Key("vfid"): tx.UUID(),
t.Key("subpath", default="."): t.String(),
}, ),
) as params:
await log_manager_api_entry(log, "get_container_mount", params)
ctx: Context = request.app["ctx"]
async with ctx.get_volume(params["volume"]) as volume:
try:
mount_path = await volume.get_vfolder_mount(
params["vfid"],
params["subpath"],
)
except VFolderNotFoundError:
raise web.HTTPBadRequest(
body=json.dumps(
{
"msg": "VFolder not found",
"vfid": str(params["vfid"]),
}, ),
content_type="application/json",
)
except InvalidSubpathError as e:
raise web.HTTPBadRequest(
body=json.dumps(
{
"msg": "Invalid vfolder subpath",
"vfid": str(params["vfid"]),
"subpath": str(e.args[1]),
}, ),
content_type="application/json",
)
return web.json_response({
"path": str(mount_path),
}, )
def test_subdict_sample(self):
def check_passwords_equal(data):
if data['password'] != data['password_confirm']:
return t.DataError('Passwords are not equal',
code='are_no_equal')
return data['password']
check_password = t.String()
passwords_key = subdict(
'password',
t.Key('password', trafaret=check_password),
t.Key('password_confirm', trafaret=check_password),
trafaret=check_passwords_equal,
)
signup_trafaret = t.Dict(
t.Key('email', trafaret=t.Email),
passwords_key,
)
res = signup_trafaret({
'email': u'*****@*****.**',
'password': u'qwerty',
'password_confirm': u'qwerty'
})
assert res == {'email': u'*****@*****.**', 'password': u'qwerty'}
res = catch_error(
signup_trafaret, {
'email': u'*****@*****.**',
'password': u'qwerty',
'password_confirm': u'not qwerty'
})
assert res.as_dict() == {'password': '******'}
res = catch_error(signup_trafaret, {
'email': u'*****@*****.**',
'password': u'qwerty'
})
assert res.as_dict() == {'password_confirm': 'is required'}
def _validator(self):
if self._model is None:
raise t.DataError('ModelValidator is not associated with model')
fields = {}
for column in self._model.__table__.columns.values():
key = t.Key(column.name, **self.key_kwargs(column))
trafaret = self.cut(column, **self.val_kwargs(column))
if trafaret is None: # chain node can return None to skip field
continue
fields[key] = trafaret
return t.Dict(fields)
async def rename_file(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict(
{
t.Key("volume"): t.String(),
t.Key("vfid"): tx.UUID(),
t.Key("relpath"): tx.PurePath(relative_only=True),
t.Key("new_name"): t.String(),
t.Key("is_dir"): t.ToBool(), # ignored since 22.03
}, ),
) as params:
await log_manager_api_entry(log, "rename_file", params)
ctx: Context = request.app["ctx"]
async with ctx.get_volume(params["volume"]) as volume:
with handle_fs_errors(volume, params["vfid"]):
await volume.move_file(
params["vfid"],
params["relpath"],
params["relpath"].with_name(params["new_name"]),
)
return web.Response(status=204)
async def tus_check_session(request: web.Request) -> web.Response:
"""
Check the availability of an upload session.
"""
ctx: Context = request.app['ctx']
secret = ctx.local_config['storage-proxy']['secret']
async with check_params(request, t.Dict({
t.Key('token'): tx.JsonWebToken(secret=secret, inner_iv=upload_token_data_iv),
}), read_from=CheckParamSource.QUERY) as params:
token_data = params['token']
async with ctx.get_volume(token_data['volume']) as volume:
headers = await prepare_tus_session_headers(request, token_data, volume)
return web.Response(headers=headers)
async def create_upload_session(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict({
t.Key('volume'): t.String(),
t.Key('vfid'): tx.UUID(),
t.Key('relpath'): tx.PurePath(relative_only=True),
t.Key('size'): t.ToInt,
})) as params:
await log_manager_api_entry(log, 'create_upload_session', params)
ctx: Context = request.app['ctx']
async with ctx.get_volume(params['volume']) as volume:
session_id = await volume.prepare_upload(params['vfid'])
token_data = {
'op':
'upload',
'volume':
params['volume'],
'vfid':
str(params['vfid']),
'relpath':
str(params['relpath']),
'size':
params['size'],
'session':
session_id,
'exp':
datetime.utcnow() +
ctx.local_config['storage-proxy']['session-expire'],
}
token = jwt.encode(
token_data,
ctx.local_config['storage-proxy']['secret'],
algorithm='HS256',
).decode('UTF-8')
return web.json_response({
'token': token,
})
async def fetch_file(request: web.Request) -> web.StreamResponse:
"""
Direct file streaming API for internal use, such as retrieving
task logs from a user vfolder ".logs".
"""
async with check_params(
request,
t.Dict(
{
t.Key("volume"): t.String(),
t.Key("vfid"): tx.UUID(),
t.Key("relpath"): tx.PurePath(relative_only=True),
}, ),
) as params:
await log_manager_api_entry(log, "fetch_file", params)
ctx: Context = request.app["ctx"]
response = web.StreamResponse(status=200)
response.headers[hdrs.CONTENT_TYPE] = "application/octet-stream"
try:
prepared = False
async with ctx.get_volume(params["volume"]) as volume:
with handle_fs_errors(volume, params["vfid"]):
async for chunk in volume.read_file(
params["vfid"],
params["relpath"],
):
if not chunk:
return response
if not prepared:
await response.prepare(request)
prepared = True
await response.write(chunk)
except FileNotFoundError:
response = web.Response(status=404, reason="Log data not found")
finally:
if prepared:
await response.write_eof()
return response
async def delete_files(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict(
{
t.Key("volume"): t.String(),
t.Key("vfid"): tx.UUID(),
t.Key("relpaths"): t.List(tx.PurePath(relative_only=True)),
t.Key("recursive", default=False): t.ToBool,
}, ),
) as params:
await log_manager_api_entry(log, "delete_files", params)
ctx: Context = request.app["ctx"]
async with ctx.get_volume(params["volume"]) as volume:
with handle_fs_errors(volume, params["vfid"]):
await volume.delete_files(
params["vfid"],
params["relpaths"],
params["recursive"],
)
return web.json_response({
"status": "ok",
}, )
async def get_vfolder_usage(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict({
t.Key("volume"): t.String(),
t.Key("vfid"): tx.UUID(),
}, ),
) as params:
try:
await log_manager_api_entry(log, "get_vfolder_usage", params)
ctx: Context = request.app["ctx"]
async with ctx.get_volume(params["volume"]) as volume:
usage = await volume.get_usage(params["vfid"])
return web.json_response(
{
"file_count": usage.file_count,
"used_bytes": usage.used_bytes,
}, )
except ExecutionError:
return web.Response(
status=500,
reason="Storage server is busy. Please try again",
)
def test_base2(self):
trafaret = t.Dict({t.Key('bar', optional=True): t.String}, foo=t.ToInt)
trafaret = trafaret.allow_extra('*')
res = trafaret.check({"foo": 1, "ham": 100, "baz": None})
assert res == {'baz': None, 'foo': 1, 'ham': 100}
res = extract_error(trafaret, {"bar": 1, "ham": 100, "baz": None})
assert res == {'bar': 'value is not a string', 'foo': 'is required'}
res = extract_error(trafaret, {
"foo": 1,
"bar": 1,
"ham": 100,
"baz": None
})
assert res == {'bar': 'value is not a string'}
async def mkdir(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict(
{
t.Key("volume"): t.String(),
t.Key("vfid"): tx.UUID(),
t.Key("relpath"): tx.PurePath(relative_only=True),
t.Key("parents", default=True): t.ToBool,
t.Key("exist_ok", default=False): t.ToBool,
}, ),
) as params:
await log_manager_api_entry(log, "mkdir", params)
ctx: Context = request.app["ctx"]
async with ctx.get_volume(params["volume"]) as volume:
with handle_fs_errors(volume, params["vfid"]):
await volume.mkdir(
params["vfid"],
params["relpath"],
parents=params["parents"],
exist_ok=params["exist_ok"],
)
return web.Response(status=204)
def test_base3(self):
trafaret = t.Dict({t.Key('bar', default='nyanya') >> 'baz': t.String},
foo=t.Int)
res = trafaret.check({'foo': 4})
self.assertEqual(res, {'baz': 'nyanya', 'foo': 4})
trafaret.allow_extra('*')
res = trafaret.check({'baz': 'spam', 'foo': 4})
self.assertEqual(res, {'baz': 'nyanya', 'foo': 4})
trafaret.ignore_extra('fooz')
res = trafaret.check({'foo': 4, 'fooz': 5})
self.assertEqual(res, {'baz': 'nyanya', 'foo': 4})
trafaret.ignore_extra('*')
res = trafaret.check({'foo': 4, 'foor': 5})
self.assertEqual(res, {'baz': 'nyanya', 'foo': 4})
class TestForward(unittest.TestCase):
FWD = T.Forward()
TRAFARET = T.Dict({
T.Key("value", optional=True): FWD,
})
FWD << T.Int()
def test_int(self):
self.assertEqual(get_err(self.TRAFARET, u"""
value: "hello"
"""), dedent(u"""\
config.yaml:2: value: value can't be converted to int
"""))
async def get_performance_metric(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict({
t.Key("volume"): t.String(),
}, ),
) as params:
await log_manager_api_entry(log, "get_performance_metric", params)
ctx: Context = request.app["ctx"]
async with ctx.get_volume(params["volume"]) as volume:
metric = await volume.get_performance_metric()
return web.json_response({
"metric": attr.asdict(metric),
}, )
class ChromeEmulationInfoTrafaret(trf.Trafaret):
scheme = trf.List(
trf.Dict(
{
"device_name": TextToEnumTrafaret(DeviceName),
trf.Key("screen_orientation", optional=True): UpperTextToEnumTrafaret(
ScreenOrientation
),
}
)
)
def check_and_return(self, value, context=None):
sanitized = self.scheme.check(value, context)
return [ChromeEmulationInfo(**dct) for dct in sanitized]
async def tus_upload_part(request: web.Request) -> web.Response:
"""
Perform the chunk upload.
"""
ctx: Context = request.app["ctx"]
secret = ctx.local_config["storage-proxy"]["secret"]
async with check_params(
request,
t.Dict(
{
t.Key("token"):
tx.JsonWebToken(
secret=secret,
inner_iv=upload_token_data_iv,
),
}, ),
read_from=CheckParamSource.QUERY,
) as params:
token_data = params["token"]
async with ctx.get_volume(token_data["volume"]) as volume:
headers = await prepare_tus_session_headers(
request, token_data, volume)
vfpath = volume.mangle_vfpath(token_data["vfid"])
upload_temp_path = vfpath / ".upload" / token_data["session"]
async with AsyncFileWriter(
target_filename=upload_temp_path,
access_mode="ab",
max_chunks=DEFAULT_INFLIGHT_CHUNKS,
) as writer:
while not request.content.at_eof():
chunk = await request.content.read(DEFAULT_CHUNK_SIZE)
await writer.write(chunk)
current_size = Path(upload_temp_path).stat().st_size
if current_size >= int(token_data["size"]):
target_path = vfpath / token_data["relpath"]
upload_temp_path.rename(target_path)
try:
loop = asyncio.get_running_loop()
await loop.run_in_executor(
None,
lambda: upload_temp_path.parent.rmdir(),
)
except OSError:
pass
headers["Upload-Offset"] = str(current_size)
return web.Response(status=204, headers=headers)
class i18nModel(BaseModel):
__collection__ = "i18ntests"
inc_id = True
structure = t.Dict({
'list_attrs':
t.List(t.String),
'salers':
t.List(t.Dict({
'name': t.String,
'address': t.String
})),
t.Key('list_sku', default=[]):
t.List(t.Int)
}).allow_extra('*')
i18n = ['list_attrs', 'salers']
indexes = [('quantity', DESCENDING), 'name']
async def get_vfolder_fs_usage(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict({
t.Key("volume"): t.String(),
}, ),
) as params:
await log_manager_api_entry(log, "get_vfolder_fs_usage", params)
ctx: Context = request.app["ctx"]
async with ctx.get_volume(params["volume"]) as volume:
fs_usage = await volume.get_fs_usage()
return web.json_response(
{
"capacity_bytes": fs_usage.capacity_bytes,
"used_bytes": fs_usage.used_bytes,
}, )
def setup_app(app: Application):
if isfile('config.yaml'):
TRAFARET = T.Dict({
T.Key('services'):
T.List(
T.Dict({
'name': T.String(),
'jwt_ttl_minutes': T.Int(),
'redirect_link': T.String(),
'algorithm': T.String(),
'secret_key': T.String()
}))
})
config = read_and_validate('config.yaml', TRAFARET)
app['services'] = {x['name']: x for x in config['services']}