def create_schema() -> T.Dict:
"""
Build schema for the configuration's file
by aggregating all the subsystem configurations
"""
# pylint: disable=protected-access
schema = T.Dict(
{
"version": T.String(),
"main": T.Dict(
{
"host": T.IP,
"port": T.ToInt(),
"client_outdir": T.String(),
"log_level": T.Enum(*logging._nameToLevel.keys()),
"testing": T.Bool(),
T.Key("studies_access_enabled", default=False): T.Or(
T.Bool(), T.ToInt
),
}
),
addon_section(tracing.tracing_section_name, optional=True): tracing.schema,
db_config.CONFIG_SECTION_NAME: db_config.schema,
director_config.CONFIG_SECTION_NAME: director_config.schema,
rest_config.CONFIG_SECTION_NAME: rest_config.schema,
projects_config.CONFIG_SECTION_NAME: projects_config.schema,
email_config.CONFIG_SECTION_NAME: email_config.schema,
storage_config.CONFIG_SECTION_NAME: storage_config.schema,
addon_section(
login_config.CONFIG_SECTION_NAME, optional=True
): login_config.schema,
addon_section(
socketio_config.CONFIG_SECTION_NAME, optional=True
): socketio_config.schema,
session_config.CONFIG_SECTION_NAME: session_config.schema,
activity_config.CONFIG_SECTION_NAME: activity_config.schema,
resource_manager_config.CONFIG_SECTION_NAME: resource_manager_config.schema,
# BELOW HERE minimal sections until more options are needed
addon_section("diagnostics", optional=True): minimal_addon_schema(),
addon_section("users", optional=True): minimal_addon_schema(),
addon_section("groups", optional=True): minimal_addon_schema(),
addon_section("tags", optional=True): minimal_addon_schema(),
addon_section("publications", optional=True): minimal_addon_schema(),
addon_section("catalog", optional=True): catalog_config.schema,
addon_section("products", optional=True): minimal_addon_schema(),
addon_section("computation", optional=True): minimal_addon_schema(),
addon_section("director-v2", optional=True): minimal_addon_schema(),
addon_section("studies_access", optional=True): minimal_addon_schema(),
addon_section("studies_dispatcher", optional=True): minimal_addon_schema(),
}
)
section_names = [k.name for k in schema.keys]
# fmt: off
assert len(section_names) == len(set(section_names)), f"Found repeated section names in {section_names}" # nosec
# fmt: on
return schema
def test_bool(self):
res = t.Bool().check(True)
self.assertEqual(res, True)
res = t.Bool().check(False)
self.assertEqual(res, False)
err = extract_error(t.Bool(), 1)
self.assertEqual(err, 'value should be True or False')
def create_schema():
"""
Build schema for the configuration's file
by aggregating all the subsystem configurations
"""
schema = T.Dict({
"version":
T.String(),
"main":
T.Dict({
"host": T.IP,
"port": T.Int(),
"log_level": T.Enum(*logging._nameToLevel.keys()), # pylint: disable=protected-access
"enabled_development_mode": T.Bool(),
}),
rest_config.CONFIG_SECTION_NAME:
rest_config.schema,
## Add here more configurations
})
section_names = [k.name for k in schema.keys]
assert len(section_names) == len(set(
section_names)), "Found repeated section names in %s" % section_names
return schema
def construct(arg):
'''
Shortcut syntax to define trafarets.
- int, str, float and bool will return t.Int, t.String, t.Float and t.Bool
- one element list will return t.List
- tuple or list with several args will return t.Tuple
- dict will return t.Dict. If key has '?' at the and it will be optional and '?' will be removed
- any callable will be t.Call
- otherwise it will be returned as is
construct is recursive and will try construct all lists, tuples and dicts args
'''
if isinstance(arg, t.Trafaret):
return arg
elif isinstance(arg, tuple) or (isinstance(arg, list) and len(arg) > 1):
return t.Tuple(*(construct(a) for a in arg))
elif isinstance(arg, list):
# if len(arg) == 1
return t.List(construct(arg[0]))
elif isinstance(arg, dict):
return t.Dict({
construct_key(key): construct(value)
for key, value in arg.items()
})
elif isinstance(arg, str):
return t.Atom(arg)
elif isinstance(arg, type):
if arg is int:
return t.Int()
elif arg is float:
return t.Float()
elif arg is str:
return t.String()
elif arg is bool:
return t.Bool()
else:
return t.Type(arg)
elif callable(arg):
return t.Call(arg)
else:
return arg
def create_schema():
"""
Build schema for the configuration's file
by aggregating all the subsystem configurations
"""
schema = T.Dict({
"version":
T.String(),
"main":
T.Dict({
"host": T.IP,
"port": T.Int(),
"client_outdir": T.String(),
"log_level": T.Enum(*logging._nameToLevel.keys()), # pylint: disable=protected-access
"testing": T.Bool(),
}),
db_config.CONFIG_SECTION_NAME:
db_config.schema,
director_config.CONFIG_SECTION_NAME:
director_config.schema,
rest_config.CONFIG_SECTION_NAME:
rest_config.schema,
projects_config.CONFIG_SECTION_NAME:
projects_config.schema,
email_config.CONFIG_SECTION_NAME:
email_config.schema,
computation_config.CONFIG_SECTION_NAME:
computation_config.schema,
storage_config.CONFIG_SECTION_NAME:
storage_config.schema,
T.Key(login_config.CONFIG_SECTION_NAME, optional=True):
login_config.schema
#s3_config.CONFIG_SECTION_NAME: s3_config.schema
#TODO: enable when sockets are refactored
})
section_names = [k.name for k in schema.keys]
assert len(section_names) == len(set(
section_names)), "Found repeated section names in %s" % section_names
return schema
import trafaret as T
from .cfg import DEFAULTS
CONFIG_SECTION_NAME = "login"
# TODO: merge with cfg.py
schema = T.Dict({
T.Key("enabled", default=True, optional=True):
T.Bool(),
T.Key(
"registration_confirmation_required",
default=DEFAULTS["REGISTRATION_CONFIRMATION_REQUIRED"],
optional=True,
):
T.Or(T.Bool, T.ToInt),
T.Key("registration_invitation_required", default=False, optional=True):
T.Or(T.Bool, T.ToInt),
})
'timeout': T.Int(),
'max_requests': T.Int(),
}),
T.Key('redis'):
T.Dict({
'host': T.String(),
'port': T.Int(),
'minsize': T.Int(),
'maxsize': T.Int(),
}),
# T.Key('host'): T.String(regex=primitive_ip_regexp),
# T.Key('port'): T.Int(),
T.Key('fernet_key'):
T.String(),
T.Key('debugtoolbar_enable'):
T.Bool(),
T.Key('pulsar_server_enable'):
T.Bool(),
T.Key('cdn'):
T.Dict({
'host': T.String(),
'process_env': T.Bool(),
}),
T.Key('qiniu'):
T.Dict({
'access_key': T.String(),
'secret_key': T.String(),
'bucket_name': T.String(),
'adir': T.String(),
'replace_adir': T.String(),
}),
def test_on_error(self):
trafaret = t.OnError(t.Bool(), message='Changed message')
res = trafaret(True)
self.assertEqual(res, True)
""" email's subsystem's configuration
- config-file schema
- settings
"""
import trafaret as T
CONFIG_SECTION_NAME = "smtp"
schema = T.Dict(
{
T.Key(
"sender", default="OSPARC support <*****@*****.**>"
): T.String(), # FIXME: email format
"host": T.String(),
"port": T.ToInt(),
T.Key("tls", default=False): T.Or(T.Bool(), T.ToInt),
T.Key("username", default=None): T.Or(T.String, T.Null),
T.Key("password", default=None): T.Or(T.String, T.Null),
}
)
def test_repr(self):
assert repr(t.Bool()) == '<Bool>'
import aiozipkin as az
import trafaret as T
from aiohttp import web
from pydantic import AnyHttpUrl, BaseSettings
log = logging.getLogger(__name__)
def setup_tracing(app: web.Application, app_name: str, host: str, port: str,
config: Dict) -> bool:
zipkin_address = f"{config['zipkin_endpoint']}/api/v2/spans"
endpoint = az.create_endpoint(app_name, ipv4=host, port=port)
loop = asyncio.get_event_loop()
tracer = loop.run_until_complete(
az.create(zipkin_address, endpoint, sample_rate=1.0))
az.setup(app, tracer)
return True
schema = T.Dict({
T.Key("enabled", default=True, optional=True):
T.Or(T.Bool(), T.ToInt),
T.Key("zipkin_endpoint", default="http://jaeger:9411"):
T.String(),
})
class TracingSettings(BaseSettings):
enabled: Optional[bool] = True
zipkin_endpoint: AnyHttpUrl = "http://jaeger:9411"
import trafaret as t
from trafaret_config import read_and_validate
from trafaret_config import ConfigError
log = logging.getLogger(__name__)
DB_ENV_TRAFARET = t.Dict({
'dsn': t.String(),
'timeout': t.Int(),
})
APP_ENV_TRAFARET = t.Dict({
'host': t.String(),
'port': t.String(),
'debug': t.Bool(),
})
CONFIG_TRAFARET = t.Dict({
'db': DB_ENV_TRAFARET,
'app': APP_ENV_TRAFARET,
})
def parse_config(path):
try:
config = read_and_validate(path, CONFIG_TRAFARET)
except ConfigError as e:
for error_message in e.errors:
log.error(str(error_message))
sys.exit(1)
def test_bool(self, check_value, result):
res = t.Bool().check(check_value)
assert res == result
import trafaret as t
STORE_LOCATION_SCHEMA = t.Enum('S3', 'gcs', 'azure', 'rackspace', 'dropbox')
STORE_SCHEMA = t.Dict({
'filename': t.String(),
'mimetype': t.String(),
'location': t.String(),
'path': t.String(),
'container': t.String(),
'region': t.String(),
'access': t.String(),
'base64decode': t.Bool(),
'workflows': t.List(t.String())
}).make_optional('*')
t.Key('product_url'): t.String()
})
lite_product_t = t.Dict({
t.Key('_id'): ObjectId,
t.Key('product_name'): t.String(),
})
products_pagination_t = t.Dict({
t.Key('total'): t.Int,
t.Key('products'): t.List(product_t)
})
wish_t = t.Dict({
t.Key('product_id'): t.String(),
t.Key('reserved'): t.Bool(),
})
extended_wish_t = t.Dict({
t.Key('_id'): ObjectId,
t.Key('product_id'): t.String(),
t.Key('reserved'): t.Bool(),
t.Key('product_name'): t.String(),
t.Key('discription'): t.String(),
t.Key('price'): t.Float(),
t.Key('img_url'): t.String(),
t.Key('product_url'): t.String()
})
extended_alien_wish_t = t.Dict({
t.Key('_id'): ObjectId,
else:
logging.basicConfig(level=default_level)
def load_settings(path):
with open(path, "rt") as f:
return yaml.safe_load(f.read())
app_config = t.Dict(
{
t.Key("chunk_size"): t.Int(),
t.Key("retries"): t.Int(),
t.Key("spacy_model"): t.String(),
t.Key("output_file"): t.String(),
t.Key("csv"): t.Bool(),
t.Key("search_terms"): t.List(t.String()),
t.Key("variable_patterns"): t.List(
t.Dict(
{
"id": t.String(),
"pattern_matchers": t.List(t.List(t.Dict({}).allow_extra("*"))),
}
)
),
t.Key("disease_patterns"): t.List(
t.Dict(
{
"id": t.String(),
"pattern_matchers": t.List(t.List(t.Dict({}).allow_extra("*"))),
}
app_config = t.Dict({
t.Key("app"):
t.Dict({
"host": t.String(),
"port": t.Int(),
}),
t.Key("redis"):
t.Dict({
"host": t.String(),
"port": t.Int(),
}),
t.Key("background_task"):
t.Dict({
"name": t.String(),
"enabled": t.Bool(),
"delay": t.Int(),
"startup_delay": t.Int(),
}),
})
def get_config() -> Any:
try:
parser = argparse.ArgumentParser(add_help=False)
required = parser.add_argument_group(
"required arguments") # noqa: F841
optional = parser.add_argument_group("optional arguments")
# Add back help
optional.add_argument(
""" socketio subsystem's configuration
- config-file schema
- settings
"""
from typing import Dict
import trafaret as T
from aiohttp import web
from servicelib.application_keys import APP_CONFIG_KEY
from socketio import AsyncServer
CONFIG_SECTION_NAME = "socketio"
APP_CLIENT_SOCKET_SERVER_KEY = __name__ + ".socketio_socketio"
APP_CLIENT_SOCKET_DECORATED_HANDLERS_KEY = __name__ + ".socketio_handlers"
schema = T.Dict({
T.Key("enabled", default=True, optional=True):
T.Or(T.Bool(), T.ToInt()),
})
def get_config(app: web.Application) -> Dict:
return app[APP_CONFIG_KEY][CONFIG_SECTION_NAME]
def get_socket_server(app: web.Application) -> AsyncServer:
return app[APP_CLIENT_SOCKET_SERVER_KEY]
T.List(
T.Dict({
"id":
T.String(),
"url":
T.URL,
T.Key("username", optional=True, default=""):
T.Or(T.String(allow_blank=True), T.Null),
T.Key("password", optional=True, default=""):
T.Or(T.String(allow_blank=True), T.Null),
T.Key("branch", default="master", optional=True):
T.String(allow_blank=True),
T.Key("tags", default="", optional=True):
T.String(allow_blank=True),
"pull_only_files":
T.Bool(),
"paths":
T.List(T.String()),
}),
min_length=1,
),
"docker_private_registries":
T.List(
T.Dict({
"url":
T.URL,
T.Key("username", optional=True, default=""):
T.String(allow_blank=True),
T.Key("password", optional=True, default=""):
T.String(allow_blank=True),
})),
""" projects subsystem's configuration
- config-file schema
- settings
"""
from typing import Dict, Optional
import trafaret as T
from aiohttp.web import Application
from pydantic import BaseSettings
from servicelib.aiohttp.application_keys import APP_CONFIG_KEY
CONFIG_SECTION_NAME = "projects"
schema = T.Dict({T.Key("enabled", default=True, optional=True): T.Bool()})
class ProjectSettings(BaseSettings):
enabled: Optional[bool] = True
def assert_valid_config(app: Application) -> Dict:
cfg = app[APP_CONFIG_KEY][CONFIG_SECTION_NAME]
_settings = ProjectSettings(**cfg)
return cfg
""" projects subsystem's configuration
- config-file schema
- settings
"""
import trafaret as T
CONFIG_SECTION_NAME = "projects"
schema = T.Dict({
T.Key("enabled", default=True, optional=True): T.Bool(),
T.Key("location"):
T.Or(T.String, T.URL), # either path or url should contain version in it
})
def __init__(self, **kwargs):
super(Boolean, self).__init__(**kwargs)
self._trafaret = t.Bool()
if self.allow_none:
self._trafaret |= t.Null()
def test_extract_error(self):
err = extract_error(t.Bool(), 1)
assert err == 'value should be True or False'
import sys
import pkg_resources
import trafaret as T
from trafaret_config import parse_and_validate, ConfigError
from baka.log import log
trafaret_yaml = T.Dict({
T.Key('package'):
T.String(),
T.Key('baka'):
T.Dict({
T.Key('debug_all', default=False):
T.Bool(),
T.Key('meta', optional=True):
T.Dict({
T.Key('version', optional=True): T.String(),
T.Key('app', optional=True): T.String(),
}),
T.Key('validator', optional=True, default=False):
T.Bool(),
}),
})
def merge_yaml(cls, config):
return cls.merge(config)
def config_yaml(registry, _yaml=None, config_file=None):
if _yaml is None:
def test_on_error(self):
trafaret = t.OnError(t.Bool(), message='Changed message')
res = trafaret(True)
assert res is True
def minimal_addon_schema() -> T.Dict:
return T.Dict({T.Key("enabled", default=True, optional=True): T.Bool()})
t.String(),
'url':
t.String(),
'maxSize':
t.Int(),
'minSize':
t.Int(),
'path':
t.String(),
'container':
t.String()
})
POLICY_SCHEMA.make_optional('*')
CONTENT_DOWNLOAD_SCHEMA = t.Dict({'dl': t.Bool(), 'cache': t.Bool()})
CONTENT_DOWNLOAD_SCHEMA.make_optional('*')
OVERWRITE_SCHEMA = t.Dict({'url': t.String(), 'base64decode': t.Bool()})
OVERWRITE_SCHEMA.make_optional('*')
METADATA_SCHEMA = t.Dict({
'size': t.Bool(),
'mimetype': t.Bool(),
'filename': t.Bool(),
'width': t.Bool(),
'height': t.Bool(),
'uploaded': t.Bool(),
'writable': t.Bool(),
import pytest
import trafaret as t
from test_app.models.db import Role
from tests import db
SCHEMA = t.Dict({
t.Key('success'): t.Bool(),
t.Key('data'): t.List(
t.Dict({
'id': t.Int(),
'role': t.Enum(Role.admin.value, Role.manager.value, Role.user.value)
})
)
})
async def test_all_user(client):
response = await client.get('/users')
assert response.status == 200
body = await response.json()
assert body['success'] is True
SCHEMA.check(body)
@pytest.mark.parametrize('params', (
{'role': Role.admin.name},
{'role': Role.manager.name},
{'role': Role.user.name},))
async def test_filter_all_user(client, params):
response = await client.get('/users', params=params)
from models_library.settings.redis import RedisConfig
from servicelib.application_keys import APP_CONFIG_KEY
CONFIG_SECTION_NAME = "resource_manager"
APP_CLIENT_REDIS_CLIENT_KEY = __name__ + ".resource_manager.redis_client"
APP_CLIENT_REDIS_LOCK_KEY = __name__ + ".resource_manager.redis_lock"
APP_CLIENT_SOCKET_REGISTRY_KEY = __name__ + ".resource_manager.registry"
APP_RESOURCE_MANAGER_TASKS_KEY = __name__ + ".resource_manager.tasks.key"
APP_GARBAGE_COLLECTOR_KEY = __name__ + ".resource_manager.garbage_collector_key"
# lock names and format strings
GUEST_USER_RC_LOCK_FORMAT = f"{__name__}:redlock:garbage_collect_user:{{user_id}}"
schema = T.Dict({
T.Key("enabled", default=True, optional=True):
T.Or(T.Bool(), T.ToInt()),
T.Key("resource_deletion_timeout_seconds", default=900, optional=True):
T.ToInt(),
T.Key("garbage_collection_interval_seconds", default=30, optional=True):
T.ToInt(),
T.Key("redis", optional=False):
T.Dict({
T.Key("enabled", default=True, optional=True): T.Bool(),
T.Key("host", default="redis", optional=True): T.String(),
T.Key("port", default=6793, optional=True): T.ToInt(),
}),
})
class RedisSection(RedisConfig):
enabled: bool = True
""" rest subsystem's configuration
- config-file schema
- settings
"""
from typing import Dict
import trafaret as T
from aiohttp import web
from servicelib.application_keys import APP_CONFIG_KEY, APP_OPENAPI_SPECS_KEY
CONFIG_SECTION_NAME = "rest"
schema = T.Dict(
{T.Key("enabled", default=True, optional=True): T.Bool(), "version": T.Enum("v0"),}
)
def get_rest_config(app: web.Application) -> Dict:
return app[APP_CONFIG_KEY][CONFIG_SECTION_NAME]
__all__ = ["APP_OPENAPI_SPECS_KEY"]
