def build_trafaret(sa_type, **kwargs):
if isinstance(sa_type, sa.sql.sqltypes.Enum):
trafaret = t.Enum(*sa_type.enums, **kwargs)
# check for Text should be before String
elif isinstance(sa_type, sa.sql.sqltypes.Text):
trafaret = t.String(**kwargs)
elif isinstance(sa_type, sa.sql.sqltypes.String):
trafaret = t.String(max_length=sa_type.length, **kwargs)
elif isinstance(sa_type, sa.sql.sqltypes.Integer):
trafaret = t.ToInt(**kwargs)
elif isinstance(sa_type, sa.sql.sqltypes.Float):
trafaret = t.ToFloat(**kwargs)
elif isinstance(sa_type, sa.sql.sqltypes.DateTime):
trafaret = DateTime(**kwargs) # RFC3339
elif isinstance(sa_type, sa.sql.sqltypes.Date):
trafaret = DateTime(**kwargs) # RFC3339
elif isinstance(sa_type, sa.sql.sqltypes.Boolean):
trafaret = t.ToBool(**kwargs)
# Add PG related JSON and ARRAY
elif isinstance(sa_type, postgresql.JSON):
trafaret = AnyDict | t.List(AnyDict)
# Add PG related JSON and ARRAY
elif isinstance(sa_type, postgresql.ARRAY):
item_trafaret = build_trafaret(sa_type.item_type)
trafaret = t.List(item_trafaret)
else:
type_ = str(sa_type)
msg = 'Validator for type {} not implemented'.format(type_)
raise NotImplementedError(msg)
return trafaret
def read_sysfs(path: Union[str, Path],
type_: Type[Any],
default: Any = None) -> Any:
def_vals: Mapping[Any, Any] = {
bool: False,
int: 0,
float: 0.0,
str: '',
}
if type_ not in def_vals:
raise TypeError('unsupported conversion type from sysfs content')
if default is None:
default = def_vals[type_]
try:
raw_str = Path(path).read_text().strip()
if type_ is bool:
return t.ToBool().check(raw_str)
else:
return type_(raw_str)
except IOError:
return default
async def rename_file(request: web.Request) -> web.Response:
async with check_params(
request,
t.Dict(
{
t.Key("volume"): t.String(),
t.Key("vfid"): tx.UUID(),
t.Key("relpath"): tx.PurePath(relative_only=True),
t.Key("new_name"): t.String(),
t.Key("is_dir"): t.ToBool(), # ignored since 22.03
}, ),
) as params:
await log_manager_api_entry(log, "rename_file", params)
ctx: Context = request.app["ctx"]
async with ctx.get_volume(params["volume"]) as volume:
with handle_fs_errors(volume, params["vfid"]):
await volume.move_file(
params["vfid"],
params["relpath"],
params["relpath"].with_name(params["new_name"]),
)
return web.Response(status=204)
def acquire_config(*, environ: t.Mapping[str, str] = os.environ) -> Config:
"""Attempt to resolve a complete Config instance from the environment.
Args:
environ: if specified a mapping to use rather than `os.environ` to
locate environment variables for configuration values.
Returns:
A complete instance of the `Config`
Raises:
trafaret.DataError if any required value is missing, or any specified
value for configuration is malformed.
"""
env_converter = tr.Dict(
{
tr.Key(
"TMPMAIL_MAIL_DOMAIN", optional=True, to_name="mail_domain",
): tr.String,
tr.Key(
"TMPMAIL_LMTP_HOST", optional=True, to_name="lmtp_host"
): tr.String(),
tr.Key("TMPMAIL_LMTP_PORT", optional=True, to_name="lmtp_port"): tr.ToInt(
gt=0, lt=(2 ** 16)
),
tr.Key(
"TMPMAIL_HTTP_HOST", optional=True, to_name="http_host"
): tr.String(),
tr.Key("TMPMAIL_HTTP_PORT", optional=True, to_name="http_port"): tr.ToInt(
gt=0, lt=(2 ** 16)
),
tr.Key(
"TMPMAIL_HTTP_HOST_STATIC", optional=True, to_name="http_host_static"
): tr.ToBool(),
},
ignore_extra="*",
)
return Config(**env_converter(environ))
def test_repr(self):
assert repr(t.ToBool()) == '<ToBool>'
def test_extract_error(self):
res = extract_error(t.ToBool(), 'aloha')
assert res == "value can't be converted to Bool"
def test_str_bool(self, value, expected_result):
actual_result = t.ToBool().check(value)
assert actual_result == expected_result
import trafaret as T
TRAFARET = T.Dict({
T.Key('database'):
T.Dict({
'db_driver': T.String(),
'database': T.String(),
'user': T.String(),
'password': T.String(),
'host': T.String(),
'port': T.Int(),
'minsize': T.Int(),
'maxsize': T.Int(),
'retry_limit': T.Int(),
'retry_interval': T.Int(),
'ssl': T.ToBool(),
'echo': T.ToBool(),
}),
T.Key('host'):
T.String(),
T.Key('port'):
T.Int(),
})
async def form_text_block(string: str) -> dict:
block = dict(type="section", text=dict(type="mrkdwn", text=string))
return block
async def form_actions_block(buttons: list) -> dict:
params['request_url'],
'request_status':
params['request_status'],
'traceback':
params['traceback'],
})
result = await conn.execute(query)
assert result.rowcount == 1
return web.json_response(resp)
@auth_required
@server_status_required(READ_ALLOWED)
@check_api_params(
t.Dict({
t.Key('mark_read', default=False): t.ToBool(),
t.Key('page_size', default=20): t.ToInt(lt=101),
t.Key('page_no', default=1): t.ToInt()
}), )
async def list_logs(request: web.Request, params: Any) -> web.Response:
resp: MutableMapping[str, Any] = {'logs': []}
dbpool = request.app['dbpool']
domain_name = request['user']['domain_name']
user_role = request['user']['role']
user_uuid = request['user']['uuid']
requester_access_key, owner_access_key = await get_access_key_scopes(
request, params)
log.info(
'LIST (ak:{0}/{1})', requester_access_key,
owner_access_key if owner_access_key != requester_access_key else '*')
async def check_presets(request: web.Request, params: Any) -> web.Response:
'''
Returns the list of all resource presets in the current scaling group,
with additional information including allocatability of each preset,
amount of total remaining resources, and the current keypair resource limits.
'''
try:
access_key = request['keypair']['access_key']
resource_policy = request['keypair']['resource_policy']
domain_name = request['user']['domain_name']
# TODO: uncomment when we implement scaling group.
# scaling_group = request.query.get('scaling_group')
# assert scaling_group is not None, 'scaling_group parameter is missing.'
except (json.decoder.JSONDecodeError, AssertionError) as e:
raise InvalidAPIParameters(extra_msg=str(e.args[0]))
registry = request.app['registry']
known_slot_types = await registry.config_server.get_resource_slots()
resp: MutableMapping[str, Any] = {
'keypair_limits': None,
'keypair_using': None,
'keypair_remaining': None,
'scaling_group_remaining': None,
'scaling_groups': None,
'presets': [],
}
log.info('CHECK_PRESETS (ak:{}, g:{}, sg:{})',
request['keypair']['access_key'], params['group'],
params['scaling_group'])
async with request.app['dbpool'].acquire() as conn, conn.begin():
# Check keypair resource limit.
keypair_limits = ResourceSlot.from_policy(resource_policy,
known_slot_types)
keypair_occupied = await registry.get_keypair_occupancy(access_key,
conn=conn)
keypair_remaining = keypair_limits - keypair_occupied
# Check group resource limit and get group_id.
j = sa.join(groups, association_groups_users,
association_groups_users.c.group_id == groups.c.id)
query = (sa.select(
[groups.c.id, groups.c.total_resource_slots]).select_from(j).where(
(association_groups_users.c.user_id == request['user']['uuid'])
& (groups.c.name == params['group'])
& (domains.c.name == domain_name)))
result = await conn.execute(query)
row = await result.fetchone()
group_id = row.id
group_resource_slots = row.total_resource_slots
if group_id is None:
raise InvalidAPIParameters('Unknown user group')
group_resource_policy = {
'total_resource_slots': group_resource_slots,
'default_for_unspecified': DefaultForUnspecified.UNLIMITED
}
group_limits = ResourceSlot.from_policy(group_resource_policy,
known_slot_types)
group_occupied = await registry.get_group_occupancy(group_id,
conn=conn)
group_remaining = group_limits - group_occupied
# Check domain resource limit.
query = (sa.select([domains.c.total_resource_slots
]).where(domains.c.name == domain_name))
domain_resource_slots = await conn.scalar(query)
domain_resource_policy = {
'total_resource_slots': domain_resource_slots,
'default_for_unspecified': DefaultForUnspecified.UNLIMITED
}
domain_limits = ResourceSlot.from_policy(domain_resource_policy,
known_slot_types)
domain_occupied = await registry.get_domain_occupancy(domain_name,
conn=conn)
domain_remaining = domain_limits - domain_occupied
# Take minimum remaining resources. There's no need to merge limits and occupied.
# To keep legacy, we just merge all remaining slots into `keypair_remainig`.
for slot in known_slot_types:
keypair_remaining[slot] = min(
keypair_remaining[slot],
group_remaining[slot],
domain_remaining[slot],
)
# Prepare per scaling group resource.
sgroups = await query_allowed_sgroups(conn, domain_name, group_id,
access_key)
sgroup_names = [sg.name for sg in sgroups]
if params['scaling_group'] is not None:
if params['scaling_group'] not in sgroup_names:
raise InvalidAPIParameters('Unknown scaling group')
sgroup_names = [params['scaling_group']]
per_sgroup = {
sgname: {
'using':
ResourceSlot({k: Decimal(0)
for k in known_slot_types.keys()}),
'remaining':
ResourceSlot({k: Decimal(0)
for k in known_slot_types.keys()}),
}
for sgname in sgroup_names
}
# Per scaling group resource using from resource occupying kernels.
query = (sa.select([
kernels.c.occupied_slots, kernels.c.scaling_group
]).select_from(kernels).where(
(kernels.c.user_uuid == request['user']['uuid'])
& (kernels.c.status.in_(AGENT_RESOURCE_OCCUPYING_KERNEL_STATUSES))
& (kernels.c.scaling_group.in_(sgroup_names))))
async for row in conn.execute(query):
per_sgroup[row.scaling_group]['using'] += row.occupied_slots
# Per scaling group resource remaining from agents stats.
sgroup_remaining = ResourceSlot(
{k: Decimal(0)
for k in known_slot_types.keys()})
query = (sa.select([
agents.c.available_slots, agents.c.occupied_slots,
agents.c.scaling_group
]).select_from(agents).where((agents.c.status == AgentStatus.ALIVE) & (
agents.c.scaling_group.in_(sgroup_names))))
agent_slots = []
async for row in conn.execute(query):
remaining = row['available_slots'] - row['occupied_slots']
remaining += ResourceSlot(
{k: Decimal(0)
for k in known_slot_types.keys()})
sgroup_remaining += remaining
agent_slots.append(remaining)
per_sgroup[row.scaling_group]['remaining'] += remaining
# Take maximum allocatable resources per sgroup.
for sgname, sgfields in per_sgroup.items():
for rtype, slots in sgfields.items():
if rtype == 'remaining':
for slot in known_slot_types.keys():
if slot in slots:
slots[slot] = min(keypair_remaining[slot],
slots[slot])
per_sgroup[sgname][rtype] = slots.to_json(
) # type: ignore # it's serialization
for slot in known_slot_types.keys():
sgroup_remaining[slot] = min(keypair_remaining[slot],
sgroup_remaining[slot])
# Fetch all resource presets in the current scaling group.
query = (sa.select([resource_presets]).select_from(resource_presets))
async for row in conn.execute(query):
# Check if there are any agent that can allocate each preset.
allocatable = False
preset_slots = row['resource_slots'].normalize_slots(
ignore_unknown=True)
for agent_slot in agent_slots:
if agent_slot >= preset_slots and keypair_remaining >= preset_slots:
allocatable = True
break
resp['presets'].append({
'name':
row['name'],
'resource_slots':
preset_slots.to_json(),
'shared_memory':
str(row['shared_memory'])
if row['shared_memory'] is not None else None,
'allocatable':
allocatable,
})
# Return group resource status as NaN if not allowed.
group_resource_visibility = await request.app[
'registry'].config_server.get(
'config/api/resources/group_resource_visibility')
group_resource_visibility = t.ToBool().check(group_resource_visibility)
if not group_resource_visibility:
group_limits = ResourceSlot(
{k: Decimal('NaN')
for k in known_slot_types.keys()})
group_occupied = ResourceSlot(
{k: Decimal('NaN')
for k in known_slot_types.keys()})
group_remaining = ResourceSlot(
{k: Decimal('NaN')
for k in known_slot_types.keys()})
resp['keypair_limits'] = keypair_limits.to_json()
resp['keypair_using'] = keypair_occupied.to_json()
resp['keypair_remaining'] = keypair_remaining.to_json()
resp['group_limits'] = group_limits.to_json()
resp['group_using'] = group_occupied.to_json()
resp['group_remaining'] = group_remaining.to_json()
resp['scaling_group_remaining'] = sgroup_remaining.to_json()
resp['scaling_groups'] = per_sgroup
return web.json_response(resp, status=200)
