async def rescan_images( self, registry: str = None, *, reporter: ProgressReporter = None, ) -> None: registry_config_iv = t.Mapping(t.String, container_registry_iv) latest_registry_config = registry_config_iv.check( await self.etcd.get_prefix('config/docker/registry')) self['docker']['registry'] = latest_registry_config # TODO: delete images from registries removed from the previous config? if registry is None: # scan all configured registries registries = self['docker']['registry'] else: try: registries = {registry: self['docker']['registry'][registry]} except KeyError: raise RuntimeError("It is an unknown registry.", registry) async with aiotools.TaskGroup() as tg: for registry_name, registry_info in registries.items(): log.info('Scanning kernel images from the registry "{0}"', registry_name) scanner_cls = get_container_registry(registry_info) scanner = scanner_cls(self.etcd, registry_name, registry_info) tg.create_task(scanner.rescan_single_registry(reporter))
def as_trafaret(cls) -> t.Trafaret: return t.Dict( { t.Key("backend"): t.String, t.Key("path"): tx.Path(type="dir"), t.Key("fsprefix", default="."): tx.PurePath(relative_only=True), t.Key("options", default=None): t.Null | t.Mapping(t.String, t.Any), }, )
class BaseModel(Model): __abstract__ = True inc_id = True structure = t.Dict({ 'name': t.String, 'quantity': t.Int, 'attrs': t.Mapping(t.String, t.Or(t.Int, t.Float, t.String)), }).allow_extra('_id', '_ns', '_int_id').ignore_extra('wrong_attr') indexes = ['id'] required_fields = ['name', 'quantity']
def test_mapping(self): trafaret = t.Mapping(t.String, t.ToInt) res = trafaret.check({u"foo": 1, u"bar": 2}) assert res == {u'bar': 2, u'foo': 1} res = extract_error(trafaret, {u"foo": 1, u"bar": None}) assert res == {u'bar': {'value': 'value is not int'}} res = extract_error(trafaret, {u"foo": 1, 2: "bar"}) assert res == {2: {'key': 'value is not a string', 'value': "value can't be converted to int"}} res = extract_error(trafaret.check, None) assert res == 'value is not a dict'
def as_trafaret(cls) -> t.Trafaret: return t.Dict({ t.Key('backend'): t.String, t.Key('path'): tx.Path(type='dir'), t.Key('fsprefix', default='.'): tx.PurePath(relative_only=True), t.Key('options', default=None): t.Null | t.Mapping(t.String, t.Any), })
def test_mapping(self): trafaret = t.Mapping(t.String, t.Int) self.assertEqual(repr(trafaret), '<Mapping(<String> => <Int>)>') res = trafaret.check({"foo": 1, "bar": 2}) self.assertEqual(res, {'bar': 2, 'foo': 1}) res = extract_error(trafaret, {"foo": 1, "bar": None}) self.assertEqual(res, {'bar': {'value': 'value is not int'}}) res = extract_error(trafaret, {"foo": 1, 2: "bar"}) self.assertEqual(res, {2: {'key': 'value is not a string', 'value': "value can't be converted to int"}}) res = extract_error(trafaret.check, None) self.assertEqual(res, 'value is not a dict')
class BaseProduct(Model): __abstract__ = True structure = t.Dict({ 'name': t.String, 'quantity': t.Int, 'attrs': t.Mapping(t.String, t.Or(t.Int, t.Float, t.String)), }).allow_extra('*') i18n = ['name', 'attrs'] indexes = ['id']
def test_2_0_regression(self): t_request = t.Dict({ t.Key('params', optional=True): t.Or(t.List(t.Any()), t.Mapping(t.AnyString(), t.Any())), }) assert t_request.check({'params': { 'aaa': 123 }}) == { 'params': { 'aaa': 123 } }
async def test_mapping(): trafaret = t.Mapping(t.String, t.ToInt & check_int) res = await (trafaret.async_check({'a': '5'})) assert res == {'a': 5} # bad key with pytest.raises(t.DataError) as res: await (trafaret.async_check({None: '5'})) assert res.value.as_dict() == {None: {"key": "value is not a string"}} # bad value with pytest.raises(t.DataError) as res: await (trafaret.async_check({'b': 'qwe'})) assert res.value.as_dict() == { 'b': { "value": "value can't be converted to int" } } # is not a dict with pytest.raises(t.DataError) as res: await (trafaret.async_check(None)) assert res.value.as_dict() == "value is not a dict"
class TestMapping(unittest.TestCase): TRAFARET = T.Dict({ T.Key("items", optional=True): T.Mapping(T.String(), T.String()), }) def test_items(self): self.assertEqual( get_err( self.TRAFARET, u""" items: a: x b: 1 3: b """), dedent(u"""\ config.yaml:4: items.b.value: value is not a string config.yaml:5: items[3].key: value is not a string """))
class TestMappingOr(unittest.TestCase): TRAFARET = T.Mapping(T.String, T.Dict({}) | T.Dict({})) if BEAUTY_ERROR: def test_items(self): self.assertEqual( get_err(self.TRAFARET, u"""test: qwe"""), dedent(u"""\ config.yaml:1: test.value.<alternative 1>: value is not a dict config.yaml:1: test.value.<alternative 2>: value is not a dict """)) else: def test_items(self): self.assertEqual( get_err(self.TRAFARET, u"""test: qwe"""), dedent(u"""\ config.yaml:1: test.value[0]: value is not a dict config.yaml:1: test.value[1]: value is not a dict """))
t.ToBool, t.Key('ssl-cert', default=None): t.Null | tx.Path(type='file'), t.Key('ssl-privkey', default=None): t.Null | tx.Path(type='file'), }), t.Key('manager'): t.Dict({ t.Key('service-addr'): tx.HostPortPair(allow_blank_host=True), t.Key('ssl-enabled'): t.ToBool, t.Key('ssl-cert', default=None): t.Null | tx.Path(type='file'), t.Key('ssl-privkey', default=None): t.Null | tx.Path(type='file'), t.Key('secret'): t.String, # used to authenticate managers }), }), t.Key('volume'): t.Mapping( t.String, VolumeInfo.as_trafaret(), # volume name -> details ), t.Key('debug'): t.Dict({ t.Key('enabled', default=False): t.ToBool, }).allow_extra('*'), }).merge(etcd_config_iv).allow_extra('*')
OptKey('eq'): SimpleType, OptKey('like'): SimpleType, }) ASC = 'ASC' DESC = 'DESC' ListQuery = t.Dict({ OptKey('_page', default=1): t.Int[1:], OptKey('_perPage', default=30): t.Int[1:], OptKey('_sortField'): t.String, OptKey('_sortDir', default=DESC): t.Enum(DESC, ASC), OptKey('_filters'): t.Mapping(t.String, Filter | SimpleType) }) LoginForm = t.Dict({ "username": t.String, "password": t.String, }) def validate_query_structure(query): """Validate query arguments in list request. :param query: mapping with pagination and filtering information """ query_dict = dict(query) filters = query_dict.pop('_filters', None)
from .auth import auth_required from .utils import check_api_params from ..manager.models.base import DataLoaderManager from ..manager.models.gql import Mutations, Queries log = BraceStyleAdapter(logging.getLogger('ai.backend.gateway.admin')) @atomic @auth_required @check_api_params( t.Dict({ t.Key('query'): t.String, t.Key('variables', default=None): t.Null | t.Mapping(t.String, t.Any), tx.AliasedKey(['operation_name', 'operationName'], default=None): t.Null | t.String, })) async def handle_gql(request: web.Request, params: Any) -> web.Response: executor = request.app['admin.gql_executor'] schema = request.app['admin.gql_schema'] manager_status = await request.app['config_server'].get_manager_status() known_slot_types = await request.app['config_server'].get_resource_slots() context = { 'config': request.app['config'], 'config_server': request.app['config_server'], 'etcd': request.app['config_server'].etcd, 'user': request['user'], 'access_key': request['keypair']['access_key'], 'dbpool': request.app['dbpool'],
import aiohttp_cors import jinja2 from aiohttp import web from uuid import uuid4 import trafaret as t clients = {} submit_message_validator = t.Dict( state=t.String, chunks=t.Mapping( t.String, t.List(t.Dict( presentationTime=t.Float, state=t.String )) ), url=t.String ) async def submit_handler(request): ws = web.WebSocketResponse() await ws.prepare(request) peername = request.transport.get_extra_info("peername") if peername is not None: client_host, client_port = peername else:
def test_mapping(self): trafaret = t.Mapping(CONTEXT_TRAFARET, CONTEXT_TRAFARET) self.assertEqual(trafaret({123: 123}, context=123), {123: 123})
'array', t.Key('items', optional=True, trafaret=ensure_list(json_schema)), t.Key('additionalItems', optional=True, trafaret=json_schema), trafaret=check_array, ), # object t.Key('propertyNames', optional=True, trafaret=deep_schema('propertyNames') & then(property_names)), subdict( 'object', t.Key('properties', optional=True, trafaret=deep_schema_mapping('properties', t.String())), t.Key('patternProperties', optional=True, trafaret=deep_schema_mapping('patternProperties', Pattern())), t.Key('additionalProperties', optional=True, trafaret=deep_schema('additionalProperties')), t.Key( 'dependencies', optional=True, trafaret=t.Mapping(t.String, unique_strings_list & required | deep_schema('dependencies')) ), trafaret=check_object, ), ) all_keywords = () + metadata + keywords + schema_keywords def validate_schema(schema, context=None): # we use `context` to provide register to deep schemas if context is None or isinstance(context, Register): register = context or Register() schema_name = schema.get('$id') or uuid4().urn schema_register = register.reg_schema(schema_name)
from . import routes CONFIG_TRAFARET = t.Dict({ t.Key("host"): t.String(), t.Key("port"): t.Int(), t.Key("relays"): t.Mapping( t.String(), t.Dict({ t.Key("chip", optional=True): t.String(), t.Key("line", optional=True): t.Int(), t.Key("name", optional=True): t.String(), t.Key("active", optional=True, default='high'): t.Enum('high', 'low'), t.Key("default", optional=True, default=0): t.Int(), })), }) class TrafaretYaml(click.Path): """ Configuration read from YAML file checked against trafaret rules. """ name = "trafaret yaml file"
def test_mapping(self): trafaret = t.Mapping(CONTEXT_TRAFARET, CONTEXT_TRAFARET) assert trafaret({123: 123}, context=123) == {123: 123}
def transform_default_baseurl(value: str, context): if value.startswith("/") and "baseurl" in context: if context["baseurl"] is not None: return "{context[baseurl]}{value}".format(context=context, value=value) return value Optional = partial(t.Key, optional=True) String = t.String(allow_blank=True) >> transform_string_replace SimpleType = String | t.ToBool | t.Bool | t.Float | t.Int | t.Null DataType = t.Mapping( t.String, SimpleType | t.List(SimpleType | t.Mapping(t.String, SimpleType | t.List(SimpleType))), ) ComplexType = SimpleType | t.List(SimpleType | DataType) | DataType AssertFuncType = t.String | t.Mapping(t.String, SimpleType | DataType) RegisterVariablesType = t.Mapping(t.String, SimpleType) RequestOptionsShortType = String >> transform_default_request RequestOptionsCompleteType = t.Dict({ "url": String >> transform_default_baseurl, t.Key("method", default="GET"): t.Enum("GET", "POST", "PUT", "OPTIONS", "DELETE", "INFO", "HEAD", "PATCH"), Optional("params"): t.Mapping(t.String, SimpleType), Optional("data"):
def __new__(cls, name, bases, dct): structure = dct.get('structure') if structure is not None: structure.allow_extra('_class', '_id', '_ns', '_int_id') # inheritance from abstract models: for model in bases: if getattr(model, '__abstract__', None) is True: if '__abstract__' not in dct: dct['__abstract__'] = False key_attrs = ['i18n', 'indexes', 'required_fields'] for attr in key_attrs: base_attrs = set(getattr(model, attr, [])) child_attrs = set(dct.get(attr, [])) dct.update({attr: list(base_attrs | child_attrs)}) if model.structure and structure is not None: base_structure = set(model.structure.keys) child_structure = set(structure.keys) structure.keys = list(base_structure | child_structure) structure.allow_any = structure.allow_any \ or model.structure.allow_any structure.ignore_any = structure.ignore_any \ or model.structure.ignore_any if not structure.allow_any: structure.extras = list( set(model.structure.extras) | set(structure.extras)) if not structure.ignore_any: structure.ignore = list( set(model.structure.ignore) | set(structure.ignore)) elif model.structure: dct['structure'] = model.structure break # change structure for translated fields: if not dct.get('__abstract__') and structure and dct.get('i18n'): for key in structure.keys[:]: if key.name in dct['i18n']: dct['structure'].keys.remove(key) dct['structure'].keys.append( t.Key(key.name, trafaret=t.Mapping(t.String, key.trafaret), default=key.default, optional=key.optional, to_name=key.to_name)) # add required_fields: if 'required_fields' in dct: required_fields = dct.get('required_fields') if dct.get('structure') is not None: optional = filter( lambda key: key.name not in dct['required_fields'], dct['structure'].keys) optional = map(operator.attrgetter('name'), optional) dct['structure'] = dct['structure'].make_optional(*optional) else: struct = dict.fromkeys(required_fields, t.Any) dct['structure'] = t.Dict(struct).allow_extra('*') return type.__new__(cls, name, bases, dct)
def test_repr(self): trafaret = t.Mapping(t.String, t.Int) assert repr(trafaret) == '<Mapping(<String> => <Int>)>'
if params['prefix']: # Flatten the returned ChainMap object for JSON serialization value = dict(await etcd.get_prefix_dict(params['key'])) else: value = await etcd.get(params['key']) return web.json_response({'result': value}) @atomic @superadmin_required @check_api_params( t.Dict({ t.Key('key'): t.String, t.Key('value'): (t.String(allow_blank=True) | t.Mapping(t.String(allow_blank=True), t.Any)), })) async def set_config(request: web.Request, params: Any) -> web.Response: shared_config: SharedConfig = request.app['shared_config'] etcd = shared_config.etcd log.info('ETCD.SET_CONFIG (ak:{}, key:{}, val:{})', request['keypair']['access_key'], params['key'], params['value']) if isinstance(params['value'], Mapping): updates = {} def flatten(prefix, o): for k, v in o.items(): inner_prefix = prefix if k == '' else f'{prefix}/{k}' if isinstance(v, Mapping): flatten(inner_prefix, v) else:
def test_mapping(self): trafaret = t.Mapping(t.String, t.Int & check_int) res = self.loop.run_until_complete(trafaret.async_check({'a': '5'})) self.assertEqual(res, {'a': 5})
}).allow_extra('*'), t.Key('api', default=_shdefs['api']): t.Dict({ t.Key('allow-origins', default=_shdefs['api']['allow-origins']): t.String, }).allow_extra('*'), t.Key('redis', default=_shdefs['redis']): t.Dict({ t.Key('addr', default=_shdefs['redis']['addr']): tx.HostPortPair, t.Key('password', default=_shdefs['redis']['password']): t.Null | t.String, }).allow_extra('*'), t.Key('docker'): t.Dict({ t.Key('registry'): t.Mapping(t.String, container_registry_iv), }).allow_extra('*'), t.Key('plugins', default=_shdefs['plugins']): t.Dict({ t.Key('accelerator', default=_shdefs['plugins']['accelerator']): t.Mapping(t.String, t.Mapping(t.String, t.Any)), t.Key('scheduler', default=_shdefs['plugins']['scheduler']): t.Mapping(t.String, t.Mapping(t.String, t.Any)), }).allow_extra('*'), t.Key('network', default=_shdefs['network']): t.Dict({ t.Key('subnet', default=_shdefs['network']['subnet']): t.Dict({ t.Key('agent', default=_shdefs['network']['subnet']['agent']): tx.IPNetwork, t.Key('container',
def validate_upload_tags(d): t.List(t.String, max_length=10).check(list(d.keys())) t.Mapping(t.String(max_length=128), t.String(max_length=256)).check(d) return d
} shared_config_iv = t.Dict({ t.Key('system', default=_shdefs['system']): t.Dict({ t.Key('timezone', default=_shdefs['system']['timezone']): tx.TimeZone, }).allow_extra('*'), t.Key('api', default=_shdefs['api']): t.Dict({ t.Key('allow-origins', default=_shdefs['api']['allow-origins']): t.String, }).allow_extra('*'), t.Key('redis', default=_shdefs['redis']): t.Dict({ t.Key('addr', default=_shdefs['redis']['addr']): tx.HostPortPair, t.Key('password', default=_shdefs['redis']['password']): t.Null | t.String, }).allow_extra('*'), t.Key('plugins', default=_shdefs['plugins']): t.Dict({ t.Key('accelerator', default=_shdefs['plugins']['accelerator']): t.Mapping(t.String, t.Mapping(t.String, t.Any)), t.Key('scheduler', default=_shdefs['plugins']['scheduler']): t.Mapping(t.String, t.Mapping(t.String, t.Any)), }).allow_extra('*'), t.Key('network', default=_shdefs['network']): t.Dict({ t.Key('subnet', default=_shdefs['network']['subnet']): t.Dict({ t.Key('agent', default=_shdefs['network']['subnet']['agent']): tx.IPNetwork, t.Key('container', default=_shdefs['network']['subnet']['container']): tx.IPNetwork, }).allow_extra('*'), }).allow_extra('*'), t.Key('watcher', default=_shdefs['watcher']): t.Dict({ t.Key('token', default=_shdefs['watcher']['token']): t.Null | t.String, }).allow_extra('*'), }).allow_extra('*')