import trafaret as T """ trafaret for server config """ TRAFARET = T.Dict({ T.Key('session_secret'): T.String(), T.Key('users'): T.Any(), T.Key('mysql'): T.Dict({ 'db': T.String(), 'host': T.String(), 'user': T.String(), 'password': T.String(allow_blank=True), 'port': T.Int(), 'minsize': T.Int(), 'maxsize': T.Int(), }), T.Key('gitlab'): T.Dict({ 'url': T.String(), 'access_token': T.String(), }), T.Key('workers'): T.Dict({ 'enable': T.Bool(), 'max_attempts': T.Int(), 'scan_timeout': T.Int(), }), T.Key('jenkins'):
def test_add_to_names_list_of_keys(self): dct = t.Dict(key1=t.String) dct + [t.Key('a', trafaret=t.String())]
- config-file schema - settings """ from typing import Dict import trafaret as T from aiohttp import ClientSession, web from servicelib.application_keys import APP_CONFIG_KEY APP_STORAGE_SESSION_KEY = __name__ + ".storage_session" CONFIG_SECTION_NAME = 'storage' schema = T.Dict({ T.Key("enabled", default=True, optional=True): T.Bool(), T.Key("host", default="storage"): T.String(), T.Key("port", default=11111): T.Int(), T.Key("version", default="v0"): T.Regexp(regexp=r'^v\d+') # storage API version basepath }) def get_config(app: web.Application) -> Dict: return app[APP_CONFIG_KEY][CONFIG_SECTION_NAME] def get_client_session(app: web.Application) -> ClientSession: return app[APP_STORAGE_SESSION_KEY]
@export.route('ics') @login_required def export_ics(): if not g.user.team.exportics_config or \ not g.user.team.exportics_config.calendar_prodid or \ not g.user.team.exportics_config.config_slug: return redirect(url_for('export.setup_ics')) return render_template('export/ics.html', config=g.user.team.exportics_config) SETUP_FORM = t.Dict({ 'calendar_prodid': t.String, 'query': t.String(allow_blank=True) }).ignore_extra('_csrf_token') @export.route('ics/setup', methods=['POST', 'GET']) @login_required def setup_ics(): g.errors = [] if request.method == 'POST': do_setup() if not g.errors: return redirect(url_for('export.export_ics')) return render_template('export/setup_ics.html', errors=g.errors, initial=g.user.team.exportics_config)
import datetime from time import struct_time, strftime, strptime, localtime import trafaret as t from bson import ObjectId from trafaret.contrib.object_id import MongoId from typing import List url_keywords = t.Dict({ t.Key('_id'): MongoId(), t.Key('url'): t.String(), t.Key('keywords'): t.List(t.String()) }) user = t.Dict({ t.Key('_id'): MongoId(), t.Key('user_ip'): t.String(), t.Key('next_trial'): t.String(), t.Key('urls'): t.List(t.String()), t.Key('last_pay_id'): t.String() }) def convert_time_to_str(time: struct_time) -> str: return strftime("%Y-%m-%d %H:%M:%S", time) def convert_str_to_time(s: str) -> struct_time: return strptime(s, "%Y-%m-%d %H:%M:%S")
def _str_col(self, column, **kwargs): return t.String(max_length=column.type.length, **kwargs)
import trafaret as T from .rest_config import schema as rest_schema app_schema = T.Dict({ T.Key("host", default="0.0.0.0"): T.IP, "port": T.Int(), "log_level": T.Enum("DEBUG", "WARNING", "INFO", "ERROR", "CRITICAL", "FATAL", "NOTSET"), "watched_git_repositories": T.List(T.Dict({ "id": T.String(), "url": T.URL, T.Key("username", optional=True, default=""): T.String(allow_blank=True), T.Key("password", optional=True, default=""): T.String(allow_blank=True), T.Key("branch", default="master", optional=True): T.String(allow_blank=True), T.Key("tags", default="", optional=True): T.String(allow_blank=True), "pull_only_files": T.Bool(), "paths": T.List(T.String()), }), min_length=1), "docker_private_registries":
from aiohttp import web from servicelib.aiohttp.application_keys import APP_CONFIG_KEY from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup from servicelib.aiohttp.tracing import setup_tracing CONFIG_SECTION_NAME = "tracing" log = logging.getLogger(__name__) # TODO: deprecated by TracingSettings in https://github.com/ITISFoundation/osparc-simcore/pull/2376 # NOT used schema = T.Dict( { T.Key("enabled", default=True, optional=True): T.Or(T.Bool(), T.ToInt), T.Key("zipkin_endpoint", default="http://jaeger:9411"): T.String(), } ) @app_module_setup(__name__, ModuleCategory.ADDON, logger=log) def setup_app_tracing(app: web.Application): # TODO: this should be part of app settings but # temporary here until https://github.com/ITISFoundation/osparc-simcore/pull/2376 is completed config = app[APP_CONFIG_KEY] host = config["main"]["host"] port = config["main"]["port"] cfg = config[CONFIG_SECTION_NAME] zipkin_endpoint = cfg["zipkin_endpoint"] assert cfg["enabled"] # nosec
class SessionResource(Resource): validation = t.Dict({ 'email': t.Email, 'password': t.Or(t.String(allow_blank=True), t.Null) }).make_optional('password').ignore_extra('*') def get(self, id=None): return jsonify_status_code(self._get_response()) @method_wrapper(http.CREATED) def post(self): data = self.clean(g.request_json) if not User.is_unique(data['email']): raise t.DataError({'email': _("This email is already taken")}) register_user(email=data['email'], password=data.get('password', '*')) return self._get_response() @method_wrapper(http.ACCEPTED) def put(self, id): cleaned_data = self.clean(g.request_data) self._authenticate(cleaned_data) return self._get_response() def delete(self, id): for key in ('identity.name', 'identity.auth_type'): session.pop(key, None) identity_changed.send(current_app._get_current_object(), identity=AnonymousIdentity()) logout_user() return jsonify_status_code(self._get_response(), http.NO_CONTENT) def clean(self, data_dict): return self.validation.check(data_dict) def _authenticate(self, data_dict): user = security.datastore.find_user(email=data_dict['email']) if verify_password(data_dict.get('password'), user.password): login_user(user) else: raise t.DataError( {'email': "Can't find anyone with this credentials"}) return data_dict def _get_response(self, **kwargs): if current_user.is_anonymous(): cuid = session.get('customer_id') else: cuid = current_user.customer.id response = { 'id': session['id'], 'is_anonymous': current_user.is_anonymous(), 'uid': session.get('user_id'), 'cuid': cuid, 'locale': g.locale } response.update(kwargs) return response
def create_schema() -> T.Dict: """ Build schema for the configuration's file by aggregating all the subsystem configurations """ # pylint: disable=protected-access schema = T.Dict({ "version": T.String(), "main": T.Dict({ "host": T.IP, "port": T.ToInt(), "log_level": T.Enum(*logging._nameToLevel.keys()), "testing": T.Bool(), T.Key("studies_access_enabled", default=False): T.Or(T.Bool(), T.ToInt), }), addon_section(tracing.CONFIG_SECTION_NAME, optional=True): tracing.schema, db__schema.CONFIG_SECTION_NAME: db__schema.schema, director__schema.CONFIG_SECTION_NAME: director__schema.schema, rest__schema.CONFIG_SECTION_NAME: rest__schema.schema, projects__schema.CONFIG_SECTION_NAME: projects__schema.schema, email__schema.CONFIG_SECTION_NAME: email__schema.schema, storage__schema.CONFIG_SECTION_NAME: storage__schema.schema, addon_section(login__schema.CONFIG_SECTION_NAME, optional=True): login__schema.schema, addon_section(socketio__schema.CONFIG_SECTION_NAME, optional=True): socketio__schema.schema, session__schema.CONFIG_SECTION_NAME: session__schema.schema, activity__schema.CONFIG_SECTION_NAME: activity__schema.schema, resource_manager__schema.CONFIG_SECTION_NAME: resource_manager__schema.schema, # BELOW HERE minimal sections until more options are needed addon_section("catalog", optional=True): catalog__schema.schema, addon_section("clusters", optional=True): minimal_addon_schema(), addon_section("computation", optional=True): minimal_addon_schema(), addon_section("diagnostics", optional=True): minimal_addon_schema(), addon_section("director-v2", optional=True): minimal_addon_schema(), addon_section("exporter", optional=True): minimal_addon_schema(), addon_section("groups", optional=True): minimal_addon_schema(), addon_section("meta_modeling", optional=True): minimal_addon_schema(), addon_section("products", optional=True): minimal_addon_schema(), addon_section("publications", optional=True): minimal_addon_schema(), addon_section("remote_debug", optional=True): minimal_addon_schema(), addon_section("security", optional=True): minimal_addon_schema(), addon_section("statics", optional=True): minimal_addon_schema(), addon_section("studies_access", optional=True): minimal_addon_schema(), addon_section("studies_dispatcher", optional=True): minimal_addon_schema(), addon_section("tags", optional=True): minimal_addon_schema(), addon_section("users", optional=True): minimal_addon_schema(), addon_section("version_control", optional=True): minimal_addon_schema(), }) section_names = [k.name for k in schema.keys] # fmt: off assert len(section_names) == len( set(section_names )), f"Found repeated section names in {section_names}" # nosec # fmt: on return schema
def test_repr(self): res = t.String() assert repr(res) == '<String>' res = t.String(allow_blank=True) assert repr(res) == '<String(blank)>'
class DeepSpeechModel: # default configuration _config_schema = t.Dict( { # path or name of the frozen model file (*.pb, *.pbmm) t.Key('model', default="data/models/deepspeech/output_graph.pbmm"): t.String(min_length=4), # Path to the language model binary file t.Key('language_model', default="data/models/deepspeech/lm.binary"): t.String(min_length=4), # trie file (build from the same vocabulary as the language model binary) # https://www.youtube.com/watch?v=-urNrIAQnNo t.Key('trie', default="data/models/deepspeech/trie"): t.String(min_length=4), # The alpha hyperparameter of the CTC decoder. Language Model weight. t.Key('lm_alpha', default=0.75): t.Float(), # The beta hyperparameter of the CTC decoder. Word insertion weight. t.Key('lm_beta', default=1.85): t.Float(), # A larger beam width value generates better results at the cost of decoding time. t.Key('beam_width', default=500): t.Int(gt=0), }, allow_extra='*') def __init__(self, **kwargs): # validate config try: self.config = self._config_schema.check(kwargs or {}) except t.DataError as err: raise ValueError('Wrong model configuration for {}: {}'.format( self, err)) self.model = None def __str__(self) -> str: return self.__class__.__name__ def __repr__(self) -> str: return '<{}>'.format(self) def __enter__(self): self.startup() return self def __exit__(self, exc_type, exc_val, exc_tb): self.shutdown() @property def log(self) -> logging.Logger: return log def startup(self): self.log.info("Starting %s ...", self) if not os.path.isfile(self.config['model']): raise FileNotFoundError(f"Invalid filename {self.config['model']}") self.model = Model(self.config['model'], self.config['beam_width']) if os.path.isfile(self.config['language_model']) and \ os.path.isfile(self.config['trie']): self.model.enableDecoderWithLM(self.config['language_model'], self.config['trie'], self.config['lm_alpha'], self.config['lm_beta']) self.log.info("Warming up %s ...", self) self.model.stt(np.zeros(160, dtype=np.int16)) def shutdown(self): """ Releases model when object deleted """ self.log.info("Shutdown %s ...", self) self.model = None self.log.info("%s Destroyed successfully", self) def process(self, audio: np.ndarray) -> str: return self.model.stt(audio)
import trafaret as t POLICY_SCHEMA = t.Dict({ 'call': t.List( t.Enum('call', 'handle', 'url', 'maxSize', 'minSize', 'path', 'container')), 'handle': t.String(), 'url': t.String(), 'maxSize': t.Int(), 'minSize': t.Int(), 'path': t.String(), 'container': t.String() }) POLICY_SCHEMA.make_optional('*') CONTENT_DOWNLOAD_SCHEMA = t.Dict({'dl': t.Bool(), 'cache': t.Bool()}) CONTENT_DOWNLOAD_SCHEMA.make_optional('*') OVERWRITE_SCHEMA = t.Dict({'url': t.String(), 'base64decode': t.Bool()}) OVERWRITE_SCHEMA.make_optional('*')
""" Basic configuration file for rabbitMQ """ from os import environ as env import pika import yaml import trafaret as T # TODO: adapt all data below! # TODO: can use venv as defaults? e.g. $RABBITMQ_LOG_CHANNEL CONFIG_SCHEMA = T.Dict({ T.Key("name", default="tasks", optional=True): T.String(), T.Key("enabled", default=True, optional=True): T.Bool(), T.Key("host", default='rabbit', optional=True): T.String(), T.Key("port", default=5672, optional=True): T.Int(), "user": T.String(), "password": T.String(), "channels": T.Dict({ "progress": T.String(), "log": T.String(),
import sys import pkg_resources import trafaret as T from trafaret_config import parse_and_validate, ConfigError from baka.log import log trafaret_yaml = T.Dict({ T.Key('package'): T.String(), T.Key('baka'): T.Dict({ T.Key('debug_all', default=False): T.Bool(), T.Key('meta', optional=True): T.Dict({ T.Key('version', optional=True): T.String(), T.Key('app', optional=True): T.String(), }), T.Key('validator', optional=True, default=False): T.Bool(), }), }) def merge_yaml(cls, config): return cls.merge(config) def config_yaml(registry, _yaml=None, config_file=None): if _yaml is None:
def test_list(self): lst = generate(t.List[t.String()]) self.assertIsInstance(lst, list) for i in lst: self.assertIsInstance(i, six.text_type)
t.String, t.Key('pid-file', default=os.devnull): tx.Path(type='file', allow_nonexisting=True, allow_devnull=True), t.Key('event-loop', default='asyncio'): t.Enum('asyncio', 'uvloop'), t.Key('skip-manager-detection', default=False): t.ToBool, }).allow_extra('*'), t.Key('container'): t.Dict({ t.Key('kernel-uid', default=-1): tx.UserID, t.Key('kernel-gid', default=-1): tx.GroupID, t.Key('kernel-host', default=''): t.String(allow_blank=True), t.Key('port-range', default=(30000, 31000)): tx.PortRange, t.Key('stats-type', default='docker'): t.Null | t.Enum(*[e.value for e in StatModes]), t.Key('sandbox-type', default='docker'): t.Enum('docker', 'jail'), t.Key('jail-args', default=[]): t.List(t.String), t.Key('scratch-type'): t.Enum('hostdir', 'memory'), t.Key('scratch-root', default='./scratches'): tx.Path(type='dir', auto_create=True), t.Key('scratch-size', default='0'): tx.BinarySize, }).allow_extra('*'),
def test_dict(self): dct = generate(t.Dict(foo=t.String(), bar=t.Int())) self.assertSetEqual(set(dct.keys()), {'foo', 'bar'}) self.assertIsInstance(dct['foo'], six.text_type) self.assertIsInstance(dct['bar'], int)
""" catalog's subsystem configuration - config-file schema - settings """ import os import trafaret as T CONFIG_SECTION_NAME = "catalog" _default_values = { "host": os.environ.get("CATALOG_HOST", "catalog"), "port": int(os.environ.get("CATALOG_PORT", 8000)), } schema = T.Dict({ T.Key("enabled", default=True, optional=True): T.Bool(), T.Key("host", default=_default_values["host"]): T.String(), T.Key("port", default=_default_values["port"]): T.ToInt(), T.Key("version", default="v0"): T.Regexp(regexp=r"^v\d+"), # catalog API version basepath })
def test_string(self): self.assertIsInstance(generate(t.String()), six.text_type)
String, Date, ) DSN = "postgresql://{user}:{password}@{host}:{port}/{database}" ADMIN_DB_URL = DSN.format(user='******', password='******', database='postgres', host='localhost', port=5432) TRAFARET = T.Dict({ T.Key('postgres'): T.Dict({ 'database': T.String(), 'user': T.String(), 'password': T.String(), 'host': T.String(), 'port': T.Int(), 'minsize': T.Int(), 'maxsize': T.Int(), }), T.Key('host'): T.IP, T.Key('port'): T.Int(), }) meta = MetaData()
import pathlib import yaml import aiofile import trafaret as t SCHEMA = t.Dict({ t.Key('id'): t.Int(), t.Key('uid'): t.String(max_length=32), t.Key('name'): t.String(), t.Key('from_template'): t.String(), t.Key('vars', optional=True): t.Dict(allow_extra='*'), t.Key('meta', optional=True): t.Dict(allow_extra='*'), t.Key('links', optional=True): t.Dict(allow_extra='*'), t.Key('other', optional=True): t.Dict(allow_extra='*') }) class AIOFileResource: def __init__(self, p: pathlib.Path): self._f = p async def read(self): async with aiofile.AIOFile(self._f.absolute().as_posix()) as afp: return await afp.read() async def write(self, data): async with aiofile.AIOFile(self._f.absolute().as_posix()) as afp: await afp.write(data) return data
if params['prefix']: # Flatten the returned ChainMap object for JSON serialization value = dict(await etcd.get_prefix_dict(params['key'])) else: value = await etcd.get(params['key']) return web.json_response({'result': value}) @atomic @admin_required @check_api_params( t.Dict({ t.Key('key'): t.String, t.Key('value'): t.Or(t.String(allow_blank=True), t.Mapping(t.String(allow_blank=True), t.Any)), })) async def set_config(request: web.Request, params: Any) -> web.Response: etcd = request.app['config_server'].etcd if isinstance(params['value'], Mapping): updates = {} def flatten(prefix, o): for k, v in o.items(): inner_prefix = prefix if k == '' else f'{prefix}/{k}' if isinstance(v, Mapping): flatten(inner_prefix, v) updates[inner_prefix] = v flatten(params['key'], params['value'])
import trafaret as t from trafaret.contrib.object_id import MongoId from trafaret.contrib.rfc_3339 import DateTime user = t.Dict({ t.Key('_id'): MongoId(), t.Key('username'): t.String(max_length=50), t.Key('email'): t.Email, t.Key('pw_hash'): t.String(), }) message = t.Dict({ t.Key('_id'): MongoId(), t.Key('author_id'): MongoId(), t.Key('username'): t.String(max_length=50), t.Key('text'): t.String(), t.Key('pub_date'): DateTime(), }) follower = t.Dict({ t.Key('_id'): MongoId(), t.Key('who_id'): MongoId(), t.Key('whom_id'): t.List(MongoId()), }) async def get_user_id(user_collection, username): rv = await (user_collection.find_one({'username': username}, {'_id': 1})) return rv['_id'] if rv else None
from typing import Any import trafaret from aiohttp import web from trafaret_config import commandline PATH = pathlib.Path(__file__).parent.parent settings_file = os.environ.get('SETTINGS_FILE') DEFAULT_CONFIG_PATH = PATH / 'config' / settings_file CONFIG_TRAFARET = trafaret.Dict({ trafaret.Key('app'): trafaret.Dict({ 'host': trafaret.String(), 'port': trafaret.Int(), }), }) def get_config(argv: Any = None) -> Any: ap = argparse.ArgumentParser() commandline.standard_argparse_options( ap, default_config=DEFAULT_CONFIG_PATH, ) options = ap.parse_args(argv) return commandline.config_from_options(options, CONFIG_TRAFARET)
import trafaret as t from trafaret.contrib.object_id import MongoId from trafaret.contrib.rfc_3339 import DateTime user = t.Dict({ t.Key('_id'): MongoId, t.Key('username'): t.String(max_length=50), t.Key('email'): t.Email, t.Key('pw_hash'): t.String, # t.Key('first_name'): t.String(max_length=50), # t.Key('last_name'): t.String(max_length=50), # t.Key('created'): DateTime, # t.Key('active'): t.Buol, }) message = t.Dict({ t.Key('_id'): MongoId, t.Key('author_id'): MongoId, t.Key('username'): t.String(max_length=50), t.Key('text'): t.String, t.Key('pub_date'): DateTime, # t.Key('likes'): t.Int, }) follower = t.Dict({ t.Key('_id'): MongoId, t.Key('who'): MongoId, t.Key('whom'): MongoId, })
def test_string(self): res = t.String() self.assertEqual(repr(res), '<String>') res = t.String(allow_blank=True) self.assertEqual(repr(res), '<String(blank)>') res = t.String().check("foo") self.assertEqual(res, 'foo') res = extract_error(t.String(), "") self.assertEqual(res, 'blank value is not allowed') res = t.String(allow_blank=True).check("") self.assertEqual(res, '') res = extract_error(t.String(), 1) self.assertEqual(res, 'value is not a string') res = t.String(regex='\w+').check('wqerwqer') self.assertEqual(res, 'wqerwqer') res = extract_error(t.String(regex='^\w+$'), 'wqe rwqer') self.assertEqual(res, "value does not match pattern: '^\\\\w+$'") res = t.String(min_length=2, max_length=3).check('123') self.assertEqual(res, '123') res = extract_error(t.String(min_length=2, max_length=6), '1') self.assertEqual(res, 'String is shorter than 2 characters') res = extract_error(t.String(min_length=2, max_length=6), '1234567') self.assertEqual(res, 'String is longer than 6 characters') # TODO # res = String(min_length=2, max_length=6, allow_blank=True) # self.assertEqual(res, Traceback (most recent call last): # ... # AssertionError: Either allow_blank or min_length should be specified, not both res = t.String(min_length=0, max_length=6, allow_blank=True).check('123') self.assertEqual(res, '123')
""" email's subsystem's configuration - config-file schema - settings """ import trafaret as T CONFIG_SECTION_NAME = "smtp" schema = T.Dict({ T.Key("sender", default="OSPARC support <*****@*****.**>"): T.String(), # FIXME: email format "host": T.String(), "port": T.ToInt(), T.Key("tls", default=False): T.Or(T.Bool(), T.ToInt), T.Key("username", default=None): T.Or(T.String, T.Null), T.Key("password", default=None): T.Or(T.String, T.Null), })
CONFIG_FILENAME = 'batch_scoring.ini' def verify_objectid(value): """Verify if id_ is a proper ObjectId. """ try: t.String(regex='^[A-Fa-f0-9]{24}$').check(value) except t.DataError: raise ValueError('id {} not a valid project/model id'.format(value)) config_validator = t.Dict({ OptKey('host'): t.String, OptKey('project_id'): t.String(regex='^[A-Fa-f0-9]{24}$'), OptKey('model_id'): t.String(regex='^[A-Fa-f0-9]{24}$'), OptKey('import_id'): t.String, OptKey('n_retry'): t.Int, OptKey('keep_cols'): t.String, OptKey('n_concurrent'): t.Int, OptKey('dataset'): t.String, OptKey('n_samples'): t.Int, OptKey('delimiter'):
import os import pathlib from typing import Any, List, Optional import trafaret from aiohttp import web from trafaret_config import commandline PATH = pathlib.Path(__file__).parent.parent settings_file = os.environ.get("SETTINGS_FILE", "api.dev.yml") DEFAULT_CONFIG_PATH = PATH / "config" / settings_file CONFIG_TRAFARET = trafaret.Dict({ trafaret.Key("app"): trafaret.Dict({ "host": trafaret.String(), "port": trafaret.Int() }), trafaret.Key("storage"): trafaret.Dict({"path": trafaret.String()}), }) def get_config(argv: Any = None) -> Any: ap = argparse.ArgumentParser() commandline.standard_argparse_options(ap, default_config=DEFAULT_CONFIG_PATH) options = ap.parse_args(argv) return commandline.config_from_options(options, CONFIG_TRAFARET)