async def rescan_images(
self,
registry: str = None,
*,
reporter: ProgressReporter = None,
) -> None:
registry_config_iv = t.Mapping(t.String, container_registry_iv)
latest_registry_config = registry_config_iv.check(
await self.etcd.get_prefix('config/docker/registry'))
self['docker']['registry'] = latest_registry_config
# TODO: delete images from registries removed from the previous config?
if registry is None:
# scan all configured registries
registries = self['docker']['registry']
else:
try:
registries = {registry: self['docker']['registry'][registry]}
except KeyError:
raise RuntimeError("It is an unknown registry.", registry)
async with aiotools.TaskGroup() as tg:
for registry_name, registry_info in registries.items():
log.info('Scanning kernel images from the registry "{0}"',
registry_name)
scanner_cls = get_container_registry(registry_info)
scanner = scanner_cls(self.etcd, registry_name, registry_info)
tg.create_task(scanner.rescan_single_registry(reporter))
def as_trafaret(cls) -> t.Trafaret:
return t.Dict(
{
t.Key("backend"): t.String,
t.Key("path"): tx.Path(type="dir"),
t.Key("fsprefix", default="."): tx.PurePath(relative_only=True),
t.Key("options", default=None): t.Null | t.Mapping(t.String, t.Any),
},
)
class BaseModel(Model):
__abstract__ = True
inc_id = True
structure = t.Dict({
'name': t.String,
'quantity': t.Int,
'attrs': t.Mapping(t.String, t.Or(t.Int, t.Float, t.String)),
}).allow_extra('_id', '_ns', '_int_id').ignore_extra('wrong_attr')
indexes = ['id']
required_fields = ['name', 'quantity']
def test_mapping(self):
trafaret = t.Mapping(t.String, t.ToInt)
res = trafaret.check({u"foo": 1, u"bar": 2})
assert res == {u'bar': 2, u'foo': 1}
res = extract_error(trafaret, {u"foo": 1, u"bar": None})
assert res == {u'bar': {'value': 'value is not int'}}
res = extract_error(trafaret, {u"foo": 1, 2: "bar"})
assert res == {2: {'key': 'value is not a string', 'value': "value can't be converted to int"}}
res = extract_error(trafaret.check, None)
assert res == 'value is not a dict'
def as_trafaret(cls) -> t.Trafaret:
return t.Dict({
t.Key('backend'):
t.String,
t.Key('path'):
tx.Path(type='dir'),
t.Key('fsprefix', default='.'):
tx.PurePath(relative_only=True),
t.Key('options', default=None):
t.Null | t.Mapping(t.String, t.Any),
})
def test_mapping(self):
trafaret = t.Mapping(t.String, t.Int)
self.assertEqual(repr(trafaret), '<Mapping(<String> => <Int>)>')
res = trafaret.check({"foo": 1, "bar": 2})
self.assertEqual(res, {'bar': 2, 'foo': 1})
res = extract_error(trafaret, {"foo": 1, "bar": None})
self.assertEqual(res, {'bar': {'value': 'value is not int'}})
res = extract_error(trafaret, {"foo": 1, 2: "bar"})
self.assertEqual(res, {2: {'key': 'value is not a string', 'value': "value can't be converted to int"}})
res = extract_error(trafaret.check, None)
self.assertEqual(res, 'value is not a dict')
class BaseProduct(Model):
__abstract__ = True
structure = t.Dict({
'name':
t.String,
'quantity':
t.Int,
'attrs':
t.Mapping(t.String, t.Or(t.Int, t.Float, t.String)),
}).allow_extra('*')
i18n = ['name', 'attrs']
indexes = ['id']
def test_2_0_regression(self):
t_request = t.Dict({
t.Key('params', optional=True):
t.Or(t.List(t.Any()), t.Mapping(t.AnyString(), t.Any())),
})
assert t_request.check({'params': {
'aaa': 123
}}) == {
'params': {
'aaa': 123
}
}
async def test_mapping():
trafaret = t.Mapping(t.String, t.ToInt & check_int)
res = await (trafaret.async_check({'a': '5'}))
assert res == {'a': 5}
# bad key
with pytest.raises(t.DataError) as res:
await (trafaret.async_check({None: '5'}))
assert res.value.as_dict() == {None: {"key": "value is not a string"}}
# bad value
with pytest.raises(t.DataError) as res:
await (trafaret.async_check({'b': 'qwe'}))
assert res.value.as_dict() == {
'b': {
"value": "value can't be converted to int"
}
}
# is not a dict
with pytest.raises(t.DataError) as res:
await (trafaret.async_check(None))
assert res.value.as_dict() == "value is not a dict"
class TestMapping(unittest.TestCase):
TRAFARET = T.Dict({
T.Key("items", optional=True):
T.Mapping(T.String(), T.String()),
})
def test_items(self):
self.assertEqual(
get_err(
self.TRAFARET, u"""
items:
a: x
b: 1
3: b
"""),
dedent(u"""\
config.yaml:4: items.b.value: value is not a string
config.yaml:5: items[3].key: value is not a string
"""))
class TestMappingOr(unittest.TestCase):
TRAFARET = T.Mapping(T.String, T.Dict({}) | T.Dict({}))
if BEAUTY_ERROR:
def test_items(self):
self.assertEqual(
get_err(self.TRAFARET, u"""test: qwe"""),
dedent(u"""\
config.yaml:1: test.value.<alternative 1>: value is not a dict
config.yaml:1: test.value.<alternative 2>: value is not a dict
"""))
else:
def test_items(self):
self.assertEqual(
get_err(self.TRAFARET, u"""test: qwe"""),
dedent(u"""\
config.yaml:1: test.value[0]: value is not a dict
config.yaml:1: test.value[1]: value is not a dict
"""))
t.ToBool,
t.Key('ssl-cert', default=None):
t.Null | tx.Path(type='file'),
t.Key('ssl-privkey', default=None):
t.Null | tx.Path(type='file'),
}),
t.Key('manager'):
t.Dict({
t.Key('service-addr'):
tx.HostPortPair(allow_blank_host=True),
t.Key('ssl-enabled'):
t.ToBool,
t.Key('ssl-cert', default=None):
t.Null | tx.Path(type='file'),
t.Key('ssl-privkey', default=None):
t.Null | tx.Path(type='file'),
t.Key('secret'):
t.String, # used to authenticate managers
}),
}),
t.Key('volume'):
t.Mapping(
t.String,
VolumeInfo.as_trafaret(), # volume name -> details
),
t.Key('debug'):
t.Dict({
t.Key('enabled', default=False): t.ToBool,
}).allow_extra('*'),
}).merge(etcd_config_iv).allow_extra('*')
OptKey('eq'): SimpleType,
OptKey('like'): SimpleType,
})
ASC = 'ASC'
DESC = 'DESC'
ListQuery = t.Dict({
OptKey('_page', default=1): t.Int[1:],
OptKey('_perPage', default=30): t.Int[1:],
OptKey('_sortField'): t.String,
OptKey('_sortDir', default=DESC): t.Enum(DESC, ASC),
OptKey('_filters'): t.Mapping(t.String, Filter | SimpleType)
})
LoginForm = t.Dict({
"username": t.String,
"password": t.String,
})
def validate_query_structure(query):
"""Validate query arguments in list request.
:param query: mapping with pagination and filtering information
"""
query_dict = dict(query)
filters = query_dict.pop('_filters', None)
from .auth import auth_required
from .utils import check_api_params
from ..manager.models.base import DataLoaderManager
from ..manager.models.gql import Mutations, Queries
log = BraceStyleAdapter(logging.getLogger('ai.backend.gateway.admin'))
@atomic
@auth_required
@check_api_params(
t.Dict({
t.Key('query'):
t.String,
t.Key('variables', default=None):
t.Null | t.Mapping(t.String, t.Any),
tx.AliasedKey(['operation_name', 'operationName'], default=None):
t.Null | t.String,
}))
async def handle_gql(request: web.Request, params: Any) -> web.Response:
executor = request.app['admin.gql_executor']
schema = request.app['admin.gql_schema']
manager_status = await request.app['config_server'].get_manager_status()
known_slot_types = await request.app['config_server'].get_resource_slots()
context = {
'config': request.app['config'],
'config_server': request.app['config_server'],
'etcd': request.app['config_server'].etcd,
'user': request['user'],
'access_key': request['keypair']['access_key'],
'dbpool': request.app['dbpool'],
import aiohttp_cors
import jinja2
from aiohttp import web
from uuid import uuid4
import trafaret as t
clients = {}
submit_message_validator = t.Dict(
state=t.String,
chunks=t.Mapping(
t.String,
t.List(t.Dict(
presentationTime=t.Float,
state=t.String
))
),
url=t.String
)
async def submit_handler(request):
ws = web.WebSocketResponse()
await ws.prepare(request)
peername = request.transport.get_extra_info("peername")
if peername is not None:
client_host, client_port = peername
else:
def test_mapping(self):
trafaret = t.Mapping(CONTEXT_TRAFARET, CONTEXT_TRAFARET)
self.assertEqual(trafaret({123: 123}, context=123), {123: 123})
'array',
t.Key('items', optional=True, trafaret=ensure_list(json_schema)),
t.Key('additionalItems', optional=True, trafaret=json_schema),
trafaret=check_array,
),
# object
t.Key('propertyNames', optional=True, trafaret=deep_schema('propertyNames') & then(property_names)),
subdict(
'object',
t.Key('properties', optional=True, trafaret=deep_schema_mapping('properties', t.String())),
t.Key('patternProperties', optional=True, trafaret=deep_schema_mapping('patternProperties', Pattern())),
t.Key('additionalProperties', optional=True, trafaret=deep_schema('additionalProperties')),
t.Key(
'dependencies',
optional=True,
trafaret=t.Mapping(t.String, unique_strings_list & required | deep_schema('dependencies'))
),
trafaret=check_object,
),
)
all_keywords = () + metadata + keywords + schema_keywords
def validate_schema(schema, context=None):
# we use `context` to provide register to deep schemas
if context is None or isinstance(context, Register):
register = context or Register()
schema_name = schema.get('$id') or uuid4().urn
schema_register = register.reg_schema(schema_name)
from . import routes
CONFIG_TRAFARET = t.Dict({
t.Key("host"):
t.String(),
t.Key("port"):
t.Int(),
t.Key("relays"):
t.Mapping(
t.String(),
t.Dict({
t.Key("chip", optional=True):
t.String(),
t.Key("line", optional=True):
t.Int(),
t.Key("name", optional=True):
t.String(),
t.Key("active", optional=True, default='high'):
t.Enum('high', 'low'),
t.Key("default", optional=True, default=0):
t.Int(),
})),
})
class TrafaretYaml(click.Path):
"""
Configuration read from YAML file checked against trafaret rules.
"""
name = "trafaret yaml file"
def test_mapping(self):
trafaret = t.Mapping(CONTEXT_TRAFARET, CONTEXT_TRAFARET)
assert trafaret({123: 123}, context=123) == {123: 123}
def transform_default_baseurl(value: str, context):
if value.startswith("/") and "baseurl" in context:
if context["baseurl"] is not None:
return "{context[baseurl]}{value}".format(context=context,
value=value)
return value
Optional = partial(t.Key, optional=True)
String = t.String(allow_blank=True) >> transform_string_replace
SimpleType = String | t.ToBool | t.Bool | t.Float | t.Int | t.Null
DataType = t.Mapping(
t.String,
SimpleType
|
t.List(SimpleType | t.Mapping(t.String, SimpleType | t.List(SimpleType))),
)
ComplexType = SimpleType | t.List(SimpleType | DataType) | DataType
AssertFuncType = t.String | t.Mapping(t.String, SimpleType | DataType)
RegisterVariablesType = t.Mapping(t.String, SimpleType)
RequestOptionsShortType = String >> transform_default_request
RequestOptionsCompleteType = t.Dict({
"url":
String >> transform_default_baseurl,
t.Key("method", default="GET"):
t.Enum("GET", "POST", "PUT", "OPTIONS", "DELETE", "INFO", "HEAD", "PATCH"),
Optional("params"):
t.Mapping(t.String, SimpleType),
Optional("data"):
def __new__(cls, name, bases, dct):
structure = dct.get('structure')
if structure is not None:
structure.allow_extra('_class', '_id', '_ns', '_int_id')
# inheritance from abstract models:
for model in bases:
if getattr(model, '__abstract__', None) is True:
if '__abstract__' not in dct:
dct['__abstract__'] = False
key_attrs = ['i18n', 'indexes', 'required_fields']
for attr in key_attrs:
base_attrs = set(getattr(model, attr, []))
child_attrs = set(dct.get(attr, []))
dct.update({attr: list(base_attrs | child_attrs)})
if model.structure and structure is not None:
base_structure = set(model.structure.keys)
child_structure = set(structure.keys)
structure.keys = list(base_structure | child_structure)
structure.allow_any = structure.allow_any \
or model.structure.allow_any
structure.ignore_any = structure.ignore_any \
or model.structure.ignore_any
if not structure.allow_any:
structure.extras = list(
set(model.structure.extras)
| set(structure.extras))
if not structure.ignore_any:
structure.ignore = list(
set(model.structure.ignore)
| set(structure.ignore))
elif model.structure:
dct['structure'] = model.structure
break
# change structure for translated fields:
if not dct.get('__abstract__') and structure and dct.get('i18n'):
for key in structure.keys[:]:
if key.name in dct['i18n']:
dct['structure'].keys.remove(key)
dct['structure'].keys.append(
t.Key(key.name,
trafaret=t.Mapping(t.String, key.trafaret),
default=key.default,
optional=key.optional,
to_name=key.to_name))
# add required_fields:
if 'required_fields' in dct:
required_fields = dct.get('required_fields')
if dct.get('structure') is not None:
optional = filter(
lambda key: key.name not in dct['required_fields'],
dct['structure'].keys)
optional = map(operator.attrgetter('name'), optional)
dct['structure'] = dct['structure'].make_optional(*optional)
else:
struct = dict.fromkeys(required_fields, t.Any)
dct['structure'] = t.Dict(struct).allow_extra('*')
return type.__new__(cls, name, bases, dct)
def test_repr(self):
trafaret = t.Mapping(t.String, t.Int)
assert repr(trafaret) == '<Mapping(<String> => <Int>)>'
if params['prefix']:
# Flatten the returned ChainMap object for JSON serialization
value = dict(await etcd.get_prefix_dict(params['key']))
else:
value = await etcd.get(params['key'])
return web.json_response({'result': value})
@atomic
@superadmin_required
@check_api_params(
t.Dict({
t.Key('key'):
t.String,
t.Key('value'): (t.String(allow_blank=True)
| t.Mapping(t.String(allow_blank=True), t.Any)),
}))
async def set_config(request: web.Request, params: Any) -> web.Response:
shared_config: SharedConfig = request.app['shared_config']
etcd = shared_config.etcd
log.info('ETCD.SET_CONFIG (ak:{}, key:{}, val:{})',
request['keypair']['access_key'], params['key'], params['value'])
if isinstance(params['value'], Mapping):
updates = {}
def flatten(prefix, o):
for k, v in o.items():
inner_prefix = prefix if k == '' else f'{prefix}/{k}'
if isinstance(v, Mapping):
flatten(inner_prefix, v)
else:
def test_mapping(self):
trafaret = t.Mapping(t.String, t.Int & check_int)
res = self.loop.run_until_complete(trafaret.async_check({'a': '5'}))
self.assertEqual(res, {'a': 5})
}).allow_extra('*'),
t.Key('api', default=_shdefs['api']):
t.Dict({
t.Key('allow-origins', default=_shdefs['api']['allow-origins']):
t.String,
}).allow_extra('*'),
t.Key('redis', default=_shdefs['redis']):
t.Dict({
t.Key('addr', default=_shdefs['redis']['addr']):
tx.HostPortPair,
t.Key('password', default=_shdefs['redis']['password']):
t.Null | t.String,
}).allow_extra('*'),
t.Key('docker'):
t.Dict({
t.Key('registry'): t.Mapping(t.String, container_registry_iv),
}).allow_extra('*'),
t.Key('plugins', default=_shdefs['plugins']):
t.Dict({
t.Key('accelerator', default=_shdefs['plugins']['accelerator']):
t.Mapping(t.String, t.Mapping(t.String, t.Any)),
t.Key('scheduler', default=_shdefs['plugins']['scheduler']):
t.Mapping(t.String, t.Mapping(t.String, t.Any)),
}).allow_extra('*'),
t.Key('network', default=_shdefs['network']):
t.Dict({
t.Key('subnet', default=_shdefs['network']['subnet']):
t.Dict({
t.Key('agent', default=_shdefs['network']['subnet']['agent']):
tx.IPNetwork,
t.Key('container',
def validate_upload_tags(d):
t.List(t.String, max_length=10).check(list(d.keys()))
t.Mapping(t.String(max_length=128), t.String(max_length=256)).check(d)
return d
}
shared_config_iv = t.Dict({
t.Key('system', default=_shdefs['system']): t.Dict({
t.Key('timezone', default=_shdefs['system']['timezone']): tx.TimeZone,
}).allow_extra('*'),
t.Key('api', default=_shdefs['api']): t.Dict({
t.Key('allow-origins', default=_shdefs['api']['allow-origins']): t.String,
}).allow_extra('*'),
t.Key('redis', default=_shdefs['redis']): t.Dict({
t.Key('addr', default=_shdefs['redis']['addr']): tx.HostPortPair,
t.Key('password', default=_shdefs['redis']['password']): t.Null | t.String,
}).allow_extra('*'),
t.Key('plugins', default=_shdefs['plugins']): t.Dict({
t.Key('accelerator', default=_shdefs['plugins']['accelerator']):
t.Mapping(t.String, t.Mapping(t.String, t.Any)),
t.Key('scheduler', default=_shdefs['plugins']['scheduler']):
t.Mapping(t.String, t.Mapping(t.String, t.Any)),
}).allow_extra('*'),
t.Key('network', default=_shdefs['network']): t.Dict({
t.Key('subnet', default=_shdefs['network']['subnet']): t.Dict({
t.Key('agent', default=_shdefs['network']['subnet']['agent']): tx.IPNetwork,
t.Key('container', default=_shdefs['network']['subnet']['container']): tx.IPNetwork,
}).allow_extra('*'),
}).allow_extra('*'),
t.Key('watcher', default=_shdefs['watcher']): t.Dict({
t.Key('token', default=_shdefs['watcher']['token']): t.Null | t.String,
}).allow_extra('*'),
}).allow_extra('*')
