Beispiel #1
0
 def test_meta(self):
     res = (t.ToInt() >> (lambda x: x * 2) >> (lambda x: x * 3)).check(1)
     assert res == 6
     res = (t.ToInt() >> float >> str).check(4)
     assert res == '4.0'
     res = (t.ToInt >> (lambda v: v if v**2 > 15 else 0)).check(5)
     assert res == 5
Beispiel #2
0
 def test_int(self):
     res = t.ToInt().check(5)
     assert res == 5
     res = extract_error(t.ToInt(), 1.1)
     assert res == 'value is not int'
     res = extract_error(t.ToInt(), 1 + 1j)
     assert res == 'value is not int'
Beispiel #3
0
    def test_class(self):
        class Tttt:
            def __call__(self, value, context=None):
                return context(value)

        trafaret = t.ToInt() & Tttt()
        assert trafaret(123, context=lambda v: v + 123) == 246
Beispiel #4
0
 def test_kwargs_ignore(self):
     trafaret = t.Dict(t.Key('foo', trafaret=t.ToInt()),
                       ignore_extra=['eggs'])
     trafaret.check({"foo": 1, "eggs": None})
     trafaret.check({"foo": 1})
     with pytest.raises(t.DataError):
         trafaret.check({"foo": 2, "marmalade": 5})
Beispiel #5
0
def create_schema() -> T.Dict:
    """
    Build schema for the configuration's file
    by aggregating all the subsystem configurations
    """
    # pylint: disable=protected-access
    schema = T.Dict(
        {
            "version": T.String(),
            "main": T.Dict(
                {
                    "host": T.IP,
                    "port": T.ToInt(),
                    "client_outdir": T.String(),
                    "log_level": T.Enum(*logging._nameToLevel.keys()),
                    "testing": T.Bool(),
                    T.Key("studies_access_enabled", default=False): T.Or(
                        T.Bool(), T.ToInt
                    ),
                }
            ),
            addon_section(tracing.tracing_section_name, optional=True): tracing.schema,
            db_config.CONFIG_SECTION_NAME: db_config.schema,
            director_config.CONFIG_SECTION_NAME: director_config.schema,
            rest_config.CONFIG_SECTION_NAME: rest_config.schema,
            projects_config.CONFIG_SECTION_NAME: projects_config.schema,
            email_config.CONFIG_SECTION_NAME: email_config.schema,
            storage_config.CONFIG_SECTION_NAME: storage_config.schema,
            addon_section(
                login_config.CONFIG_SECTION_NAME, optional=True
            ): login_config.schema,
            addon_section(
                socketio_config.CONFIG_SECTION_NAME, optional=True
            ): socketio_config.schema,
            session_config.CONFIG_SECTION_NAME: session_config.schema,
            activity_config.CONFIG_SECTION_NAME: activity_config.schema,
            resource_manager_config.CONFIG_SECTION_NAME: resource_manager_config.schema,
            # BELOW HERE minimal sections until more options are needed
            addon_section("diagnostics", optional=True): minimal_addon_schema(),
            addon_section("users", optional=True): minimal_addon_schema(),
            addon_section("groups", optional=True): minimal_addon_schema(),
            addon_section("tags", optional=True): minimal_addon_schema(),
            addon_section("publications", optional=True): minimal_addon_schema(),
            addon_section("catalog", optional=True): catalog_config.schema,
            addon_section("products", optional=True): minimal_addon_schema(),
            addon_section("computation", optional=True): minimal_addon_schema(),
            addon_section("director-v2", optional=True): minimal_addon_schema(),
            addon_section("studies_access", optional=True): minimal_addon_schema(),
            addon_section("studies_dispatcher", optional=True): minimal_addon_schema(),
        }
    )

    section_names = [k.name for k in schema.keys]

    # fmt: off
    assert len(section_names) == len(set(section_names)), f"Found repeated section names in {section_names}"  # nosec
    # fmt: on

    return schema
Beispiel #6
0
def acquire_config(*, environ: t.Mapping[str, str] = os.environ) -> Config:
    """Attempt to resolve a complete Config instance from the environment.

    Args:
        environ: if specified a mapping to use rather than `os.environ` to
            locate environment variables for configuration values.

    Returns:
        A complete instance of the `Config`

    Raises:
        trafaret.DataError if any required value is missing, or any specified
            value for configuration is malformed.
    """
    env_converter = tr.Dict(
        {
            tr.Key(
                "TMPMAIL_MAIL_DOMAIN", optional=True, to_name="mail_domain",
            ): tr.String,
            tr.Key(
                "TMPMAIL_LMTP_HOST", optional=True, to_name="lmtp_host"
            ): tr.String(),
            tr.Key("TMPMAIL_LMTP_PORT", optional=True, to_name="lmtp_port"): tr.ToInt(
                gt=0, lt=(2 ** 16)
            ),
            tr.Key(
                "TMPMAIL_HTTP_HOST", optional=True, to_name="http_host"
            ): tr.String(),
            tr.Key("TMPMAIL_HTTP_PORT", optional=True, to_name="http_port"): tr.ToInt(
                gt=0, lt=(2 ** 16)
            ),
            tr.Key(
                "TMPMAIL_HTTP_HOST_STATIC", optional=True, to_name="http_host_static"
            ): tr.ToBool(),
        },
        ignore_extra="*",
    )
    return Config(**env_converter(environ))
Beispiel #7
0
class TestToInt:
    @pytest.mark.parametrize('value, expected', [
        (0, 0),
        (1, 1),
        (1.0, 1),
        ('1', 1),
        (-1, -1),
        ('-1', -1),
    ])
    def test_to_int(self, value, expected):
        result = t.ToInt().check(value)
        assert result == expected

    @pytest.mark.parametrize('value, expected', [
        (None, WRONG_TYPE),
        ('', IS_NOT_A_NUMBER),
        ('1.0', IS_NOT_A_NUMBER),
    ])
    def test_error_code(self, value, expected):
        with pytest.raises(t.DataError) as err:
            t.ToInt().check(value)
        assert err.value.code == expected

    def test_extract_error(self):
        result = t.extract_error(t.ToInt(), '')
        assert result == 'value can\'t be converted to int'

    @pytest.mark.parametrize('value, expected', [
        (t.ToInt(), '<ToInt>'),
        (t.ToInt[1:], '<ToInt(gte=1)>'),
        (t.ToInt[1:10], '<ToInt(gte=1, lte=10)>'),
        (t.ToInt[:10], '<ToInt(lte=10)>'),
        (t.ToInt >= 3, '<ToInt(gte=3)>'),
    ])
    def test_repr(self, value, expected):
        assert repr(value) == expected

    def test_meta_res(self):
        res = (t.ToInt > 5).check(10)
        assert res == 10
        res = t.extract_error(t.ToInt > 5, 1)
        assert res == 'value should be greater than 5'
        res = (t.ToInt < 3).check(1)
        assert res == 1
        res = t.extract_error(t.ToInt < 3, 3)
        assert res == 'value should be less than 3'
        res = t.extract_error(t.ToInt >= 5, 1)
        assert res == 'value is less than 5'
        res = t.extract_error(t.ToInt <= 3, 4)
        assert res == 'value is greater than 3'
Beispiel #8
0
def construct(arg):
    '''
    Shortcut syntax to define trafarets.

    - int, str, float and bool will return t.Int, t.String, t.Float and t.Bool
    - one element list will return t.List
    - tuple or list with several args will return t.Tuple
    - dict will return t.Dict. If key has '?' at the and it will be optional and '?' will be removed
    - any callable will be t.Call
    - otherwise it will be returned as is

    construct is recursive and will try construct all lists, tuples and dicts args
    '''
    if isinstance(arg, t.Trafaret):
        return arg
    elif isinstance(arg, tuple) or (isinstance(arg, list) and len(arg) > 1):
        return t.Tuple(*(construct(a) for a in arg))
    elif isinstance(arg, list):
        # if len(arg) == 1
        return t.List(construct(arg[0]))
    elif isinstance(arg, dict):
        return t.Dict({
            construct_key(key): construct(value)
            for key, value in arg.items()
        })
    elif isinstance(arg, str):
        return t.Atom(arg)
    elif isinstance(arg, type):
        if arg is int:
            return t.ToInt()
        elif arg is float:
            return t.ToFloat()
        elif arg is str:
            return t.String()
        elif arg is bool:
            return t.Bool()
        else:
            return t.Type(arg)
    elif callable(arg):
        return t.Call(arg)
    else:
        return arg
Beispiel #9
0
def build_trafaret(sa_type, **kwargs):

    if isinstance(sa_type, sa.sql.sqltypes.Enum):
        trafaret = t.Enum(*sa_type.enums, **kwargs)

    # check for Text should be before String
    elif isinstance(sa_type, sa.sql.sqltypes.Text):
        trafaret = t.String(**kwargs)

    elif isinstance(sa_type, sa.sql.sqltypes.String):
        trafaret = t.String(max_length=sa_type.length, **kwargs)

    elif isinstance(sa_type, sa.sql.sqltypes.Integer):
        trafaret = t.ToInt(**kwargs)

    elif isinstance(sa_type, sa.sql.sqltypes.Float):
        trafaret = t.ToFloat(**kwargs)

    elif isinstance(sa_type, sa.sql.sqltypes.DateTime):
        trafaret = DateTime(**kwargs)  # RFC3339

    elif isinstance(sa_type, sa.sql.sqltypes.Date):
        trafaret = DateTime(**kwargs)  # RFC3339

    elif isinstance(sa_type, sa.sql.sqltypes.Boolean):
        trafaret = t.ToBool(**kwargs)

    # Add PG related JSON and ARRAY
    elif isinstance(sa_type, postgresql.JSON):
        trafaret = AnyDict | t.List(AnyDict)

    # Add PG related JSON and ARRAY
    elif isinstance(sa_type, postgresql.ARRAY):
        item_trafaret = build_trafaret(sa_type.item_type)
        trafaret = t.List(item_trafaret)

    else:
        type_ = str(sa_type)
        msg = 'Validator for type {} not implemented'.format(type_)
        raise NotImplementedError(msg)
    return trafaret
Beispiel #10
0
async def start_lookup(message: types.Message):
    logging.info(f'Received message {message.text}')
    response = ''
    m_split = message.text.strip().split(' ')
    if len(m_split) == 1:
        range_ = 500
    else:
        try:
            range_ = t.ToInt(gte=1, lte=5000).check(m_split[1].strip())
        except t.DataError as e:
            logging.error(e)
            range_ = 500
            response = '반경이 너무 크거나 작아요. 기본값인 500미터로 고정할게요.\n'
    response += '이 메세지의 답변 메세지로 현재 위치를 보내주세요.'
    sent_message = await bot.send_message(
        message.chat.id,
        response,
        reply_to_message_id=message.message_id,
        reply_markup=types.ForceReply(selective=True))
    store_range_info[(
        sent_message.message_id,
        message.chat.id,
    )] = range_
Beispiel #11
0
 def test_to_int(self, value, expected):
     result = t.ToInt().check(value)
     assert result == expected
Beispiel #12
0
""" Basic configuration file for postgres service

"""
from os import environ as env

import trafaret as T

CONFIG_SCHEMA = T.Dict({
    "database": T.String(),
    "user": T.String(),
    "password": T.String(),
    T.Key("minsize", default=1 ,optional=True): T.ToInt(),
    T.Key("maxsize", default=4, optional=True): T.ToInt(),
    "host": T.Or( T.String, T.Null),
    "port": T.Or( T.ToInt, T.Null),
    "endpoint": T.Or( T.String, T.Null)
})


# TODO: deprecate!
class Config():

    def __init__(self):
        # TODO: uniform config classes . see server.config file
        POSTGRES_URL = env.get("POSTGRES_ENDPOINT", "postgres:5432")
        POSTGRES_USER = env.get("POSTGRES_USER", "simcore")
        POSTGRES_PW = env.get("POSTGRES_PASSWORD", "simcore")
        POSTGRES_DB = env.get("POSTGRES_DB", "simcoredb")

        self._user = POSTGRES_USER
        self._pwd = POSTGRES_PW
Beispiel #13
0
import trafaret as T
from aiohttp import ClientSession, web
from servicelib.application_keys import APP_CONFIG_KEY, APP_CLIENT_SESSION_KEY
from yarl import URL

APP_DIRECTOR_API_KEY = __name__ + ".director_api"

CONFIG_SECTION_NAME = "director"

schema = T.Dict({
    T.Key("enabled", default=True, optional=True): T.Bool(),
    T.Key(
        "host",
        default="director",
    ): T.String(),
    T.Key("port", default=8001): T.ToInt(),
    T.Key("version", default="v0"):
    T.Regexp(regexp=r"^v\d+"),  # storage API version basepath
})


def build_api_url(config: Dict) -> URL:
    api_baseurl = URL.build(scheme="http",
                            host=config["host"],
                            port=config["port"]).with_path(config["version"])
    return api_baseurl


def get_config(app: web.Application) -> Dict:
    return app[APP_CONFIG_KEY][CONFIG_SECTION_NAME]
from pydantic import BaseSettings
from servicelib.application_keys import APP_CONFIG_KEY

from typing import Dict

from aiohttp.web import Application

CONFIG_SECTION_NAME = "smtp"

schema = T.Dict({
    T.Key("sender", default="OSPARC support <*****@*****.**>"):
    T.String(),  # FIXME: email format
    "host":
    T.String(),
    "port":
    T.ToInt(),
    T.Key("tls", default=False):
    T.Or(T.Bool(), T.ToInt),
    T.Key("username", default=None):
    T.Or(T.String, T.Null),
    T.Key("password", default=None):
    T.Or(T.String, T.Null),
})


class EmailSettings(BaseSettings):
    sender: str = "OSPARC support <*****@*****.**>"
    host: str
    port: PortInt
    tls: bool = False
    username: Optional[str] = None
Beispiel #15
0
 def test_error_code(self, value, expected):
     with pytest.raises(t.DataError) as err:
         t.ToInt().check(value)
     assert err.value.code == expected
Beispiel #16
0
"""
from typing import Dict, Optional

import trafaret as T
from aiohttp.web import Application
from pydantic import BaseSettings
from servicelib.application_keys import APP_CONFIG_KEY
from socketio import AsyncServer

CONFIG_SECTION_NAME = "socketio"
APP_CLIENT_SOCKET_SERVER_KEY = __name__ + ".socketio_socketio"
APP_CLIENT_SOCKET_DECORATED_HANDLERS_KEY = __name__ + ".socketio_handlers"

schema = T.Dict(
    {
        T.Key("enabled", default=True, optional=True): T.Or(T.Bool(), T.ToInt()),
    }
)


class SocketIOSettings(BaseSettings):
    enabled: Optional[bool] = True


def get_config(app: Application) -> Dict:
    return app[APP_CONFIG_KEY][CONFIG_SECTION_NAME]


def get_socket_server(app: Application) -> AsyncServer:
    return app[APP_CLIENT_SOCKET_SERVER_KEY]
Beispiel #17
0
            'request_status':
            params['request_status'],
            'traceback':
            params['traceback'],
        })
        result = await conn.execute(query)
        assert result.rowcount == 1
    return web.json_response(resp)


@auth_required
@server_status_required(READ_ALLOWED)
@check_api_params(
    t.Dict({
        t.Key('mark_read', default=False): t.ToBool(),
        t.Key('page_size', default=20): t.ToInt(lt=101),
        t.Key('page_no', default=1): t.ToInt()
    }), )
async def list_logs(request: web.Request, params: Any) -> web.Response:
    resp: MutableMapping[str, Any] = {'logs': []}
    dbpool = request.app['dbpool']
    domain_name = request['user']['domain_name']
    user_role = request['user']['role']
    user_uuid = request['user']['uuid']

    requester_access_key, owner_access_key = await get_access_key_scopes(
        request, params)
    log.info(
        'LIST (ak:{0}/{1})', requester_access_key,
        owner_access_key if owner_access_key != requester_access_key else '*')
    async with dbpool.acquire() as conn:
Beispiel #18
0
"""
from os import environ as env
import logging

import trafaret as T

log = logging.getLogger(__name__)


CONFIG_SCHEMA = T.Dict({
    "endpoint": T.String(),
    "access_key": T.String(),
    "secret_key": T.String(),
    "bucket_name": T.String(),
    T.Key("secure", default=0): T.ToInt(),
})


# TODO: deprecate!
class Config():
    def __init__(self):
        # TODO: uniform config classes . see server.config file
        S3_ENDPOINT = env.get("S3_ENDPOINT", "minio:9000")
        S3_ACCESS_KEY = env.get("S3_ACCESS_KEY", "12345678")
        S3_SECRET_KEY = env.get("S3_SECRET_KEY", "12345678")
        S3_BUCKET_NAME = env.get("S3_BUCKET_NAME", "simcore")
        S3_SECURE = env.get("S3_SECURE", "0")

        self._endpoint = S3_ENDPOINT
        self._access_key = S3_ACCESS_KEY
Beispiel #19
0
""" storage subsystem's configuration

    - config-file schema
    - settings
"""

import trafaret as T

CONFIG_SECTION_NAME = "storage"

schema = T.Dict({
    T.Key("enabled", default=True, optional=True): T.Bool(),
    T.Key("host", default="storage"): T.String(),
    T.Key("port", default=11111): T.ToInt(),
    T.Key("version", default="v0"):
    T.Regexp(regexp=r"^v\d+"),  # storage API version basepath
})
Beispiel #20
0
import trafaret as T

from servicelib.config_schema_utils import addon_section
from servicelib.tracing import schema as tracing_schema
from simcore_sdk.config import db, s3

from . import rest_config

runs_in_container = "SC_BUILD_TARGET" in os.environ

app_schema = T.Dict(
    {
        T.Key(
            "host", default="0.0.0.0" if runs_in_container else "127.0.0.1"  # nosec
        ): T.IP,
        "port": T.ToInt(),
        "log_level": T.Enum(
            "DEBUG", "WARNING", "INFO", "ERROR", "CRITICAL", "FATAL", "NOTSET"
        ),
        "testing": T.Bool(),
        T.Key("max_workers", default=8, optional=True): T.ToInt(),
        T.Key("monitoring_enabled", default=False): T.Or(
            T.Bool(), T.ToInt
        ),  # Int added to use environs
        T.Key("test_datcore", optional=True): T.Dict(
            {"token_key": T.String(), "token_secret": T.String()}
        ),
        T.Key("disable_services", default=[], optional=True): T.List(T.String()),
    }
)
Beispiel #21
0
 def test_extract_error(self):
     result = t.extract_error(t.ToInt(), '')
     assert result == 'value can\'t be converted to int'
Beispiel #22
0
""" email's subsystem's configuration

    - config-file schema
    - settings
"""
import trafaret as T

CONFIG_SECTION_NAME = "smtp"


schema = T.Dict(
    {
        T.Key(
            "sender", default="OSPARC support <*****@*****.**>"
        ): T.String(),  # FIXME: email format
        "host": T.String(),
        "port": T.ToInt(),
        T.Key("tls", default=False): T.Or(T.Bool(), T.ToInt),
        T.Key("username", default=None): T.Or(T.String, T.Null),
        T.Key("password", default=None): T.Or(T.String, T.Null),
    }
)
Beispiel #23
0
""" socketio subsystem's configuration

    - config-file schema
    - settings
"""
from typing import Dict

import trafaret as T
from aiohttp import web

from servicelib.application_keys import APP_CONFIG_KEY
from socketio import AsyncServer

CONFIG_SECTION_NAME = "socketio"
APP_CLIENT_SOCKET_SERVER_KEY = __name__ + ".socketio_socketio"
APP_CLIENT_SOCKET_DECORATED_HANDLERS_KEY = __name__ + ".socketio_handlers"

schema = T.Dict({
    T.Key("enabled", default=True, optional=True):
    T.Or(T.Bool(), T.ToInt()),
})


def get_config(app: web.Application) -> Dict:
    return app[APP_CONFIG_KEY][CONFIG_SECTION_NAME]


def get_socket_server(app: web.Application) -> AsyncServer:
    return app[APP_CLIENT_SOCKET_SERVER_KEY]
Beispiel #24
0
 def test_repr(self):
     assert repr(t.ToInt()) == '<ToInt>'
Beispiel #25
0
import trafaret as t

from citizens_dwh_api.constants import DATE_FORMAT


def _build_trafaret(schema, optional=False):
    return t.Dict(
        {t.Key(key, optional=optional): item
         for key, item in schema.items()})


OptionalCitizenSchema = {
    "town": t.String(min_length=1),
    "street": t.String(min_length=1),
    "building": t.String(min_length=1),
    "apartment": t.ToInt(gte=0),
    "name": t.String(min_length=1),
    "birth_date": t.ToDateTime(
        format=DATE_FORMAT  # datetime, because pymongo cannot insert date
    ),
    "gender": t.Enum("male", "female"),
    "relatives": t.List(t.ToInt(gte=0)),
}

CitizenSchema = {**OptionalCitizenSchema, **{"citizen_id": t.ToInt(gte=0)}}

OptionalCitizen = _build_trafaret(OptionalCitizenSchema, optional=True)
Citizen = _build_trafaret(CitizenSchema, optional=False)
Beispiel #26
0
from models_library.settings.redis import RedisConfig
from servicelib.application_keys import APP_CONFIG_KEY

CONFIG_SECTION_NAME = "resource_manager"
APP_CLIENT_REDIS_CLIENT_KEY = __name__ + ".resource_manager.redis_client"
APP_CLIENT_REDIS_LOCK_KEY = __name__ + ".resource_manager.redis_lock"
APP_CLIENT_SOCKET_REGISTRY_KEY = __name__ + ".resource_manager.registry"
APP_RESOURCE_MANAGER_TASKS_KEY = __name__ + ".resource_manager.tasks.key"
APP_GARBAGE_COLLECTOR_KEY = __name__ + ".resource_manager.garbage_collector_key"

# lock names and format strings
GUEST_USER_RC_LOCK_FORMAT = f"{__name__}:redlock:garbage_collect_user:{{user_id}}"

schema = T.Dict({
    T.Key("enabled", default=True, optional=True):
    T.Or(T.Bool(), T.ToInt()),
    T.Key("resource_deletion_timeout_seconds", default=900, optional=True):
    T.ToInt(),
    T.Key("garbage_collection_interval_seconds", default=30, optional=True):
    T.ToInt(),
    T.Key("redis", optional=False):
    T.Dict({
        T.Key("enabled", default=True, optional=True): T.Bool(),
        T.Key("host", default="redis", optional=True): T.String(),
        T.Key("port", default=6793, optional=True): T.ToInt(),
    }),
})


class RedisSection(RedisConfig):
    enabled: bool = True
Beispiel #27
0
from pydantic import BaseSettings, Field

from models_library.basic_types import PortInt, VersionTag
from servicelib.application_keys import APP_CLIENT_SESSION_KEY, APP_CONFIG_KEY

CONFIG_SECTION_NAME = "catalog"

_default_values = {
    "host": os.environ.get("CATALOG_HOST", "catalog"),
    "port": int(os.environ.get("CATALOG_PORT", 8000)),
}

schema = T.Dict({
    T.Key("enabled", default=True, optional=True): T.Bool(),
    T.Key("host", default=_default_values["host"]): T.String(),
    T.Key("port", default=_default_values["port"]): T.ToInt(),
    T.Key("version", default="v0"):
    T.Regexp(regexp=r"^v\d+"),  # catalog API version basepath
})


class CatalogSettings(BaseSettings):
    enabled: bool = True
    host: str = "catalog"
    port: PortInt = 8000
    vtag: VersionTag = Field("v0",
                             alias="version",
                             description="Catalog service API's version tag")

    class Config:
        prefix = "CATALOG_"