def create_schema() -> T.Dict: """ Build schema for the configuration's file by aggregating all the subsystem configurations """ # pylint: disable=protected-access schema = T.Dict( { "version": T.String(), "main": T.Dict( { "host": T.IP, "port": T.ToInt(), "client_outdir": T.String(), "log_level": T.Enum(*logging._nameToLevel.keys()), "testing": T.Bool(), T.Key("studies_access_enabled", default=False): T.Or( T.Bool(), T.ToInt ), } ), addon_section(tracing.tracing_section_name, optional=True): tracing.schema, db_config.CONFIG_SECTION_NAME: db_config.schema, director_config.CONFIG_SECTION_NAME: director_config.schema, rest_config.CONFIG_SECTION_NAME: rest_config.schema, projects_config.CONFIG_SECTION_NAME: projects_config.schema, email_config.CONFIG_SECTION_NAME: email_config.schema, storage_config.CONFIG_SECTION_NAME: storage_config.schema, addon_section( login_config.CONFIG_SECTION_NAME, optional=True ): login_config.schema, addon_section( socketio_config.CONFIG_SECTION_NAME, optional=True ): socketio_config.schema, session_config.CONFIG_SECTION_NAME: session_config.schema, activity_config.CONFIG_SECTION_NAME: activity_config.schema, resource_manager_config.CONFIG_SECTION_NAME: resource_manager_config.schema, # BELOW HERE minimal sections until more options are needed addon_section("diagnostics", optional=True): minimal_addon_schema(), addon_section("users", optional=True): minimal_addon_schema(), addon_section("groups", optional=True): minimal_addon_schema(), addon_section("tags", optional=True): minimal_addon_schema(), addon_section("publications", optional=True): minimal_addon_schema(), addon_section("catalog", optional=True): catalog_config.schema, addon_section("products", optional=True): minimal_addon_schema(), addon_section("computation", optional=True): minimal_addon_schema(), addon_section("director-v2", optional=True): minimal_addon_schema(), addon_section("studies_access", optional=True): minimal_addon_schema(), addon_section("studies_dispatcher", optional=True): minimal_addon_schema(), } ) section_names = [k.name for k in schema.keys] # fmt: off assert len(section_names) == len(set(section_names)), f"Found repeated section names in {section_names}" # nosec # fmt: on return schema
def test_bool(self): res = t.Bool().check(True) self.assertEqual(res, True) res = t.Bool().check(False) self.assertEqual(res, False) err = extract_error(t.Bool(), 1) self.assertEqual(err, 'value should be True or False')
def create_schema(): """ Build schema for the configuration's file by aggregating all the subsystem configurations """ schema = T.Dict({ "version": T.String(), "main": T.Dict({ "host": T.IP, "port": T.Int(), "log_level": T.Enum(*logging._nameToLevel.keys()), # pylint: disable=protected-access "enabled_development_mode": T.Bool(), }), rest_config.CONFIG_SECTION_NAME: rest_config.schema, ## Add here more configurations }) section_names = [k.name for k in schema.keys] assert len(section_names) == len(set( section_names)), "Found repeated section names in %s" % section_names return schema
def construct(arg): ''' Shortcut syntax to define trafarets. - int, str, float and bool will return t.Int, t.String, t.Float and t.Bool - one element list will return t.List - tuple or list with several args will return t.Tuple - dict will return t.Dict. If key has '?' at the and it will be optional and '?' will be removed - any callable will be t.Call - otherwise it will be returned as is construct is recursive and will try construct all lists, tuples and dicts args ''' if isinstance(arg, t.Trafaret): return arg elif isinstance(arg, tuple) or (isinstance(arg, list) and len(arg) > 1): return t.Tuple(*(construct(a) for a in arg)) elif isinstance(arg, list): # if len(arg) == 1 return t.List(construct(arg[0])) elif isinstance(arg, dict): return t.Dict({ construct_key(key): construct(value) for key, value in arg.items() }) elif isinstance(arg, str): return t.Atom(arg) elif isinstance(arg, type): if arg is int: return t.Int() elif arg is float: return t.Float() elif arg is str: return t.String() elif arg is bool: return t.Bool() else: return t.Type(arg) elif callable(arg): return t.Call(arg) else: return arg
def create_schema(): """ Build schema for the configuration's file by aggregating all the subsystem configurations """ schema = T.Dict({ "version": T.String(), "main": T.Dict({ "host": T.IP, "port": T.Int(), "client_outdir": T.String(), "log_level": T.Enum(*logging._nameToLevel.keys()), # pylint: disable=protected-access "testing": T.Bool(), }), db_config.CONFIG_SECTION_NAME: db_config.schema, director_config.CONFIG_SECTION_NAME: director_config.schema, rest_config.CONFIG_SECTION_NAME: rest_config.schema, projects_config.CONFIG_SECTION_NAME: projects_config.schema, email_config.CONFIG_SECTION_NAME: email_config.schema, computation_config.CONFIG_SECTION_NAME: computation_config.schema, storage_config.CONFIG_SECTION_NAME: storage_config.schema, T.Key(login_config.CONFIG_SECTION_NAME, optional=True): login_config.schema #s3_config.CONFIG_SECTION_NAME: s3_config.schema #TODO: enable when sockets are refactored }) section_names = [k.name for k in schema.keys] assert len(section_names) == len(set( section_names)), "Found repeated section names in %s" % section_names return schema
import trafaret as T from .cfg import DEFAULTS CONFIG_SECTION_NAME = "login" # TODO: merge with cfg.py schema = T.Dict({ T.Key("enabled", default=True, optional=True): T.Bool(), T.Key( "registration_confirmation_required", default=DEFAULTS["REGISTRATION_CONFIRMATION_REQUIRED"], optional=True, ): T.Or(T.Bool, T.ToInt), T.Key("registration_invitation_required", default=False, optional=True): T.Or(T.Bool, T.ToInt), })
'timeout': T.Int(), 'max_requests': T.Int(), }), T.Key('redis'): T.Dict({ 'host': T.String(), 'port': T.Int(), 'minsize': T.Int(), 'maxsize': T.Int(), }), # T.Key('host'): T.String(regex=primitive_ip_regexp), # T.Key('port'): T.Int(), T.Key('fernet_key'): T.String(), T.Key('debugtoolbar_enable'): T.Bool(), T.Key('pulsar_server_enable'): T.Bool(), T.Key('cdn'): T.Dict({ 'host': T.String(), 'process_env': T.Bool(), }), T.Key('qiniu'): T.Dict({ 'access_key': T.String(), 'secret_key': T.String(), 'bucket_name': T.String(), 'adir': T.String(), 'replace_adir': T.String(), }),
def test_on_error(self): trafaret = t.OnError(t.Bool(), message='Changed message') res = trafaret(True) self.assertEqual(res, True)
""" email's subsystem's configuration - config-file schema - settings """ import trafaret as T CONFIG_SECTION_NAME = "smtp" schema = T.Dict( { T.Key( "sender", default="OSPARC support <*****@*****.**>" ): T.String(), # FIXME: email format "host": T.String(), "port": T.ToInt(), T.Key("tls", default=False): T.Or(T.Bool(), T.ToInt), T.Key("username", default=None): T.Or(T.String, T.Null), T.Key("password", default=None): T.Or(T.String, T.Null), } )
def test_repr(self): assert repr(t.Bool()) == '<Bool>'
import aiozipkin as az import trafaret as T from aiohttp import web from pydantic import AnyHttpUrl, BaseSettings log = logging.getLogger(__name__) def setup_tracing(app: web.Application, app_name: str, host: str, port: str, config: Dict) -> bool: zipkin_address = f"{config['zipkin_endpoint']}/api/v2/spans" endpoint = az.create_endpoint(app_name, ipv4=host, port=port) loop = asyncio.get_event_loop() tracer = loop.run_until_complete( az.create(zipkin_address, endpoint, sample_rate=1.0)) az.setup(app, tracer) return True schema = T.Dict({ T.Key("enabled", default=True, optional=True): T.Or(T.Bool(), T.ToInt), T.Key("zipkin_endpoint", default="http://jaeger:9411"): T.String(), }) class TracingSettings(BaseSettings): enabled: Optional[bool] = True zipkin_endpoint: AnyHttpUrl = "http://jaeger:9411"
import trafaret as t from trafaret_config import read_and_validate from trafaret_config import ConfigError log = logging.getLogger(__name__) DB_ENV_TRAFARET = t.Dict({ 'dsn': t.String(), 'timeout': t.Int(), }) APP_ENV_TRAFARET = t.Dict({ 'host': t.String(), 'port': t.String(), 'debug': t.Bool(), }) CONFIG_TRAFARET = t.Dict({ 'db': DB_ENV_TRAFARET, 'app': APP_ENV_TRAFARET, }) def parse_config(path): try: config = read_and_validate(path, CONFIG_TRAFARET) except ConfigError as e: for error_message in e.errors: log.error(str(error_message)) sys.exit(1)
def test_bool(self, check_value, result): res = t.Bool().check(check_value) assert res == result
import trafaret as t STORE_LOCATION_SCHEMA = t.Enum('S3', 'gcs', 'azure', 'rackspace', 'dropbox') STORE_SCHEMA = t.Dict({ 'filename': t.String(), 'mimetype': t.String(), 'location': t.String(), 'path': t.String(), 'container': t.String(), 'region': t.String(), 'access': t.String(), 'base64decode': t.Bool(), 'workflows': t.List(t.String()) }).make_optional('*')
t.Key('product_url'): t.String() }) lite_product_t = t.Dict({ t.Key('_id'): ObjectId, t.Key('product_name'): t.String(), }) products_pagination_t = t.Dict({ t.Key('total'): t.Int, t.Key('products'): t.List(product_t) }) wish_t = t.Dict({ t.Key('product_id'): t.String(), t.Key('reserved'): t.Bool(), }) extended_wish_t = t.Dict({ t.Key('_id'): ObjectId, t.Key('product_id'): t.String(), t.Key('reserved'): t.Bool(), t.Key('product_name'): t.String(), t.Key('discription'): t.String(), t.Key('price'): t.Float(), t.Key('img_url'): t.String(), t.Key('product_url'): t.String() }) extended_alien_wish_t = t.Dict({ t.Key('_id'): ObjectId,
else: logging.basicConfig(level=default_level) def load_settings(path): with open(path, "rt") as f: return yaml.safe_load(f.read()) app_config = t.Dict( { t.Key("chunk_size"): t.Int(), t.Key("retries"): t.Int(), t.Key("spacy_model"): t.String(), t.Key("output_file"): t.String(), t.Key("csv"): t.Bool(), t.Key("search_terms"): t.List(t.String()), t.Key("variable_patterns"): t.List( t.Dict( { "id": t.String(), "pattern_matchers": t.List(t.List(t.Dict({}).allow_extra("*"))), } ) ), t.Key("disease_patterns"): t.List( t.Dict( { "id": t.String(), "pattern_matchers": t.List(t.List(t.Dict({}).allow_extra("*"))), }
app_config = t.Dict({ t.Key("app"): t.Dict({ "host": t.String(), "port": t.Int(), }), t.Key("redis"): t.Dict({ "host": t.String(), "port": t.Int(), }), t.Key("background_task"): t.Dict({ "name": t.String(), "enabled": t.Bool(), "delay": t.Int(), "startup_delay": t.Int(), }), }) def get_config() -> Any: try: parser = argparse.ArgumentParser(add_help=False) required = parser.add_argument_group( "required arguments") # noqa: F841 optional = parser.add_argument_group("optional arguments") # Add back help optional.add_argument(
""" socketio subsystem's configuration - config-file schema - settings """ from typing import Dict import trafaret as T from aiohttp import web from servicelib.application_keys import APP_CONFIG_KEY from socketio import AsyncServer CONFIG_SECTION_NAME = "socketio" APP_CLIENT_SOCKET_SERVER_KEY = __name__ + ".socketio_socketio" APP_CLIENT_SOCKET_DECORATED_HANDLERS_KEY = __name__ + ".socketio_handlers" schema = T.Dict({ T.Key("enabled", default=True, optional=True): T.Or(T.Bool(), T.ToInt()), }) def get_config(app: web.Application) -> Dict: return app[APP_CONFIG_KEY][CONFIG_SECTION_NAME] def get_socket_server(app: web.Application) -> AsyncServer: return app[APP_CLIENT_SOCKET_SERVER_KEY]
T.List( T.Dict({ "id": T.String(), "url": T.URL, T.Key("username", optional=True, default=""): T.Or(T.String(allow_blank=True), T.Null), T.Key("password", optional=True, default=""): T.Or(T.String(allow_blank=True), T.Null), T.Key("branch", default="master", optional=True): T.String(allow_blank=True), T.Key("tags", default="", optional=True): T.String(allow_blank=True), "pull_only_files": T.Bool(), "paths": T.List(T.String()), }), min_length=1, ), "docker_private_registries": T.List( T.Dict({ "url": T.URL, T.Key("username", optional=True, default=""): T.String(allow_blank=True), T.Key("password", optional=True, default=""): T.String(allow_blank=True), })),
""" projects subsystem's configuration - config-file schema - settings """ from typing import Dict, Optional import trafaret as T from aiohttp.web import Application from pydantic import BaseSettings from servicelib.aiohttp.application_keys import APP_CONFIG_KEY CONFIG_SECTION_NAME = "projects" schema = T.Dict({T.Key("enabled", default=True, optional=True): T.Bool()}) class ProjectSettings(BaseSettings): enabled: Optional[bool] = True def assert_valid_config(app: Application) -> Dict: cfg = app[APP_CONFIG_KEY][CONFIG_SECTION_NAME] _settings = ProjectSettings(**cfg) return cfg
""" projects subsystem's configuration - config-file schema - settings """ import trafaret as T CONFIG_SECTION_NAME = "projects" schema = T.Dict({ T.Key("enabled", default=True, optional=True): T.Bool(), T.Key("location"): T.Or(T.String, T.URL), # either path or url should contain version in it })
def __init__(self, **kwargs): super(Boolean, self).__init__(**kwargs) self._trafaret = t.Bool() if self.allow_none: self._trafaret |= t.Null()
def test_extract_error(self): err = extract_error(t.Bool(), 1) assert err == 'value should be True or False'
import sys import pkg_resources import trafaret as T from trafaret_config import parse_and_validate, ConfigError from baka.log import log trafaret_yaml = T.Dict({ T.Key('package'): T.String(), T.Key('baka'): T.Dict({ T.Key('debug_all', default=False): T.Bool(), T.Key('meta', optional=True): T.Dict({ T.Key('version', optional=True): T.String(), T.Key('app', optional=True): T.String(), }), T.Key('validator', optional=True, default=False): T.Bool(), }), }) def merge_yaml(cls, config): return cls.merge(config) def config_yaml(registry, _yaml=None, config_file=None): if _yaml is None:
def test_on_error(self): trafaret = t.OnError(t.Bool(), message='Changed message') res = trafaret(True) assert res is True
def minimal_addon_schema() -> T.Dict: return T.Dict({T.Key("enabled", default=True, optional=True): T.Bool()})
t.String(), 'url': t.String(), 'maxSize': t.Int(), 'minSize': t.Int(), 'path': t.String(), 'container': t.String() }) POLICY_SCHEMA.make_optional('*') CONTENT_DOWNLOAD_SCHEMA = t.Dict({'dl': t.Bool(), 'cache': t.Bool()}) CONTENT_DOWNLOAD_SCHEMA.make_optional('*') OVERWRITE_SCHEMA = t.Dict({'url': t.String(), 'base64decode': t.Bool()}) OVERWRITE_SCHEMA.make_optional('*') METADATA_SCHEMA = t.Dict({ 'size': t.Bool(), 'mimetype': t.Bool(), 'filename': t.Bool(), 'width': t.Bool(), 'height': t.Bool(), 'uploaded': t.Bool(), 'writable': t.Bool(),
import pytest import trafaret as t from test_app.models.db import Role from tests import db SCHEMA = t.Dict({ t.Key('success'): t.Bool(), t.Key('data'): t.List( t.Dict({ 'id': t.Int(), 'role': t.Enum(Role.admin.value, Role.manager.value, Role.user.value) }) ) }) async def test_all_user(client): response = await client.get('/users') assert response.status == 200 body = await response.json() assert body['success'] is True SCHEMA.check(body) @pytest.mark.parametrize('params', ( {'role': Role.admin.name}, {'role': Role.manager.name}, {'role': Role.user.name},)) async def test_filter_all_user(client, params): response = await client.get('/users', params=params)
from models_library.settings.redis import RedisConfig from servicelib.application_keys import APP_CONFIG_KEY CONFIG_SECTION_NAME = "resource_manager" APP_CLIENT_REDIS_CLIENT_KEY = __name__ + ".resource_manager.redis_client" APP_CLIENT_REDIS_LOCK_KEY = __name__ + ".resource_manager.redis_lock" APP_CLIENT_SOCKET_REGISTRY_KEY = __name__ + ".resource_manager.registry" APP_RESOURCE_MANAGER_TASKS_KEY = __name__ + ".resource_manager.tasks.key" APP_GARBAGE_COLLECTOR_KEY = __name__ + ".resource_manager.garbage_collector_key" # lock names and format strings GUEST_USER_RC_LOCK_FORMAT = f"{__name__}:redlock:garbage_collect_user:{{user_id}}" schema = T.Dict({ T.Key("enabled", default=True, optional=True): T.Or(T.Bool(), T.ToInt()), T.Key("resource_deletion_timeout_seconds", default=900, optional=True): T.ToInt(), T.Key("garbage_collection_interval_seconds", default=30, optional=True): T.ToInt(), T.Key("redis", optional=False): T.Dict({ T.Key("enabled", default=True, optional=True): T.Bool(), T.Key("host", default="redis", optional=True): T.String(), T.Key("port", default=6793, optional=True): T.ToInt(), }), }) class RedisSection(RedisConfig): enabled: bool = True
""" rest subsystem's configuration - config-file schema - settings """ from typing import Dict import trafaret as T from aiohttp import web from servicelib.application_keys import APP_CONFIG_KEY, APP_OPENAPI_SPECS_KEY CONFIG_SECTION_NAME = "rest" schema = T.Dict( {T.Key("enabled", default=True, optional=True): T.Bool(), "version": T.Enum("v0"),} ) def get_rest_config(app: web.Application) -> Dict: return app[APP_CONFIG_KEY][CONFIG_SECTION_NAME] __all__ = ["APP_OPENAPI_SPECS_KEY"]