def test_meta(self):
res = (t.ToInt() >> (lambda x: x * 2) >> (lambda x: x * 3)).check(1)
assert res == 6
res = (t.ToInt() >> float >> str).check(4)
assert res == '4.0'
res = (t.ToInt >> (lambda v: v if v**2 > 15 else 0)).check(5)
assert res == 5
def test_int(self):
res = t.ToInt().check(5)
assert res == 5
res = extract_error(t.ToInt(), 1.1)
assert res == 'value is not int'
res = extract_error(t.ToInt(), 1 + 1j)
assert res == 'value is not int'
def test_class(self):
class Tttt:
def __call__(self, value, context=None):
return context(value)
trafaret = t.ToInt() & Tttt()
assert trafaret(123, context=lambda v: v + 123) == 246
def test_kwargs_ignore(self):
trafaret = t.Dict(t.Key('foo', trafaret=t.ToInt()),
ignore_extra=['eggs'])
trafaret.check({"foo": 1, "eggs": None})
trafaret.check({"foo": 1})
with pytest.raises(t.DataError):
trafaret.check({"foo": 2, "marmalade": 5})
def create_schema() -> T.Dict:
"""
Build schema for the configuration's file
by aggregating all the subsystem configurations
"""
# pylint: disable=protected-access
schema = T.Dict(
{
"version": T.String(),
"main": T.Dict(
{
"host": T.IP,
"port": T.ToInt(),
"client_outdir": T.String(),
"log_level": T.Enum(*logging._nameToLevel.keys()),
"testing": T.Bool(),
T.Key("studies_access_enabled", default=False): T.Or(
T.Bool(), T.ToInt
),
}
),
addon_section(tracing.tracing_section_name, optional=True): tracing.schema,
db_config.CONFIG_SECTION_NAME: db_config.schema,
director_config.CONFIG_SECTION_NAME: director_config.schema,
rest_config.CONFIG_SECTION_NAME: rest_config.schema,
projects_config.CONFIG_SECTION_NAME: projects_config.schema,
email_config.CONFIG_SECTION_NAME: email_config.schema,
storage_config.CONFIG_SECTION_NAME: storage_config.schema,
addon_section(
login_config.CONFIG_SECTION_NAME, optional=True
): login_config.schema,
addon_section(
socketio_config.CONFIG_SECTION_NAME, optional=True
): socketio_config.schema,
session_config.CONFIG_SECTION_NAME: session_config.schema,
activity_config.CONFIG_SECTION_NAME: activity_config.schema,
resource_manager_config.CONFIG_SECTION_NAME: resource_manager_config.schema,
# BELOW HERE minimal sections until more options are needed
addon_section("diagnostics", optional=True): minimal_addon_schema(),
addon_section("users", optional=True): minimal_addon_schema(),
addon_section("groups", optional=True): minimal_addon_schema(),
addon_section("tags", optional=True): minimal_addon_schema(),
addon_section("publications", optional=True): minimal_addon_schema(),
addon_section("catalog", optional=True): catalog_config.schema,
addon_section("products", optional=True): minimal_addon_schema(),
addon_section("computation", optional=True): minimal_addon_schema(),
addon_section("director-v2", optional=True): minimal_addon_schema(),
addon_section("studies_access", optional=True): minimal_addon_schema(),
addon_section("studies_dispatcher", optional=True): minimal_addon_schema(),
}
)
section_names = [k.name for k in schema.keys]
# fmt: off
assert len(section_names) == len(set(section_names)), f"Found repeated section names in {section_names}" # nosec
# fmt: on
return schema
def acquire_config(*, environ: t.Mapping[str, str] = os.environ) -> Config:
"""Attempt to resolve a complete Config instance from the environment.
Args:
environ: if specified a mapping to use rather than `os.environ` to
locate environment variables for configuration values.
Returns:
A complete instance of the `Config`
Raises:
trafaret.DataError if any required value is missing, or any specified
value for configuration is malformed.
"""
env_converter = tr.Dict(
{
tr.Key(
"TMPMAIL_MAIL_DOMAIN", optional=True, to_name="mail_domain",
): tr.String,
tr.Key(
"TMPMAIL_LMTP_HOST", optional=True, to_name="lmtp_host"
): tr.String(),
tr.Key("TMPMAIL_LMTP_PORT", optional=True, to_name="lmtp_port"): tr.ToInt(
gt=0, lt=(2 ** 16)
),
tr.Key(
"TMPMAIL_HTTP_HOST", optional=True, to_name="http_host"
): tr.String(),
tr.Key("TMPMAIL_HTTP_PORT", optional=True, to_name="http_port"): tr.ToInt(
gt=0, lt=(2 ** 16)
),
tr.Key(
"TMPMAIL_HTTP_HOST_STATIC", optional=True, to_name="http_host_static"
): tr.ToBool(),
},
ignore_extra="*",
)
return Config(**env_converter(environ))
class TestToInt:
@pytest.mark.parametrize('value, expected', [
(0, 0),
(1, 1),
(1.0, 1),
('1', 1),
(-1, -1),
('-1', -1),
])
def test_to_int(self, value, expected):
result = t.ToInt().check(value)
assert result == expected
@pytest.mark.parametrize('value, expected', [
(None, WRONG_TYPE),
('', IS_NOT_A_NUMBER),
('1.0', IS_NOT_A_NUMBER),
])
def test_error_code(self, value, expected):
with pytest.raises(t.DataError) as err:
t.ToInt().check(value)
assert err.value.code == expected
def test_extract_error(self):
result = t.extract_error(t.ToInt(), '')
assert result == 'value can\'t be converted to int'
@pytest.mark.parametrize('value, expected', [
(t.ToInt(), '<ToInt>'),
(t.ToInt[1:], '<ToInt(gte=1)>'),
(t.ToInt[1:10], '<ToInt(gte=1, lte=10)>'),
(t.ToInt[:10], '<ToInt(lte=10)>'),
(t.ToInt >= 3, '<ToInt(gte=3)>'),
])
def test_repr(self, value, expected):
assert repr(value) == expected
def test_meta_res(self):
res = (t.ToInt > 5).check(10)
assert res == 10
res = t.extract_error(t.ToInt > 5, 1)
assert res == 'value should be greater than 5'
res = (t.ToInt < 3).check(1)
assert res == 1
res = t.extract_error(t.ToInt < 3, 3)
assert res == 'value should be less than 3'
res = t.extract_error(t.ToInt >= 5, 1)
assert res == 'value is less than 5'
res = t.extract_error(t.ToInt <= 3, 4)
assert res == 'value is greater than 3'
def construct(arg):
'''
Shortcut syntax to define trafarets.
- int, str, float and bool will return t.Int, t.String, t.Float and t.Bool
- one element list will return t.List
- tuple or list with several args will return t.Tuple
- dict will return t.Dict. If key has '?' at the and it will be optional and '?' will be removed
- any callable will be t.Call
- otherwise it will be returned as is
construct is recursive and will try construct all lists, tuples and dicts args
'''
if isinstance(arg, t.Trafaret):
return arg
elif isinstance(arg, tuple) or (isinstance(arg, list) and len(arg) > 1):
return t.Tuple(*(construct(a) for a in arg))
elif isinstance(arg, list):
# if len(arg) == 1
return t.List(construct(arg[0]))
elif isinstance(arg, dict):
return t.Dict({
construct_key(key): construct(value)
for key, value in arg.items()
})
elif isinstance(arg, str):
return t.Atom(arg)
elif isinstance(arg, type):
if arg is int:
return t.ToInt()
elif arg is float:
return t.ToFloat()
elif arg is str:
return t.String()
elif arg is bool:
return t.Bool()
else:
return t.Type(arg)
elif callable(arg):
return t.Call(arg)
else:
return arg
def build_trafaret(sa_type, **kwargs):
if isinstance(sa_type, sa.sql.sqltypes.Enum):
trafaret = t.Enum(*sa_type.enums, **kwargs)
# check for Text should be before String
elif isinstance(sa_type, sa.sql.sqltypes.Text):
trafaret = t.String(**kwargs)
elif isinstance(sa_type, sa.sql.sqltypes.String):
trafaret = t.String(max_length=sa_type.length, **kwargs)
elif isinstance(sa_type, sa.sql.sqltypes.Integer):
trafaret = t.ToInt(**kwargs)
elif isinstance(sa_type, sa.sql.sqltypes.Float):
trafaret = t.ToFloat(**kwargs)
elif isinstance(sa_type, sa.sql.sqltypes.DateTime):
trafaret = DateTime(**kwargs) # RFC3339
elif isinstance(sa_type, sa.sql.sqltypes.Date):
trafaret = DateTime(**kwargs) # RFC3339
elif isinstance(sa_type, sa.sql.sqltypes.Boolean):
trafaret = t.ToBool(**kwargs)
# Add PG related JSON and ARRAY
elif isinstance(sa_type, postgresql.JSON):
trafaret = AnyDict | t.List(AnyDict)
# Add PG related JSON and ARRAY
elif isinstance(sa_type, postgresql.ARRAY):
item_trafaret = build_trafaret(sa_type.item_type)
trafaret = t.List(item_trafaret)
else:
type_ = str(sa_type)
msg = 'Validator for type {} not implemented'.format(type_)
raise NotImplementedError(msg)
return trafaret
async def start_lookup(message: types.Message):
logging.info(f'Received message {message.text}')
response = ''
m_split = message.text.strip().split(' ')
if len(m_split) == 1:
range_ = 500
else:
try:
range_ = t.ToInt(gte=1, lte=5000).check(m_split[1].strip())
except t.DataError as e:
logging.error(e)
range_ = 500
response = '반경이 너무 크거나 작아요. 기본값인 500미터로 고정할게요.\n'
response += '이 메세지의 답변 메세지로 현재 위치를 보내주세요.'
sent_message = await bot.send_message(
message.chat.id,
response,
reply_to_message_id=message.message_id,
reply_markup=types.ForceReply(selective=True))
store_range_info[(
sent_message.message_id,
message.chat.id,
)] = range_
def test_to_int(self, value, expected):
result = t.ToInt().check(value)
assert result == expected
""" Basic configuration file for postgres service
"""
from os import environ as env
import trafaret as T
CONFIG_SCHEMA = T.Dict({
"database": T.String(),
"user": T.String(),
"password": T.String(),
T.Key("minsize", default=1 ,optional=True): T.ToInt(),
T.Key("maxsize", default=4, optional=True): T.ToInt(),
"host": T.Or( T.String, T.Null),
"port": T.Or( T.ToInt, T.Null),
"endpoint": T.Or( T.String, T.Null)
})
# TODO: deprecate!
class Config():
def __init__(self):
# TODO: uniform config classes . see server.config file
POSTGRES_URL = env.get("POSTGRES_ENDPOINT", "postgres:5432")
POSTGRES_USER = env.get("POSTGRES_USER", "simcore")
POSTGRES_PW = env.get("POSTGRES_PASSWORD", "simcore")
POSTGRES_DB = env.get("POSTGRES_DB", "simcoredb")
self._user = POSTGRES_USER
self._pwd = POSTGRES_PW
import trafaret as T
from aiohttp import ClientSession, web
from servicelib.application_keys import APP_CONFIG_KEY, APP_CLIENT_SESSION_KEY
from yarl import URL
APP_DIRECTOR_API_KEY = __name__ + ".director_api"
CONFIG_SECTION_NAME = "director"
schema = T.Dict({
T.Key("enabled", default=True, optional=True): T.Bool(),
T.Key(
"host",
default="director",
): T.String(),
T.Key("port", default=8001): T.ToInt(),
T.Key("version", default="v0"):
T.Regexp(regexp=r"^v\d+"), # storage API version basepath
})
def build_api_url(config: Dict) -> URL:
api_baseurl = URL.build(scheme="http",
host=config["host"],
port=config["port"]).with_path(config["version"])
return api_baseurl
def get_config(app: web.Application) -> Dict:
return app[APP_CONFIG_KEY][CONFIG_SECTION_NAME]
from pydantic import BaseSettings
from servicelib.application_keys import APP_CONFIG_KEY
from typing import Dict
from aiohttp.web import Application
CONFIG_SECTION_NAME = "smtp"
schema = T.Dict({
T.Key("sender", default="OSPARC support <*****@*****.**>"):
T.String(), # FIXME: email format
"host":
T.String(),
"port":
T.ToInt(),
T.Key("tls", default=False):
T.Or(T.Bool(), T.ToInt),
T.Key("username", default=None):
T.Or(T.String, T.Null),
T.Key("password", default=None):
T.Or(T.String, T.Null),
})
class EmailSettings(BaseSettings):
sender: str = "OSPARC support <*****@*****.**>"
host: str
port: PortInt
tls: bool = False
username: Optional[str] = None
def test_error_code(self, value, expected):
with pytest.raises(t.DataError) as err:
t.ToInt().check(value)
assert err.value.code == expected
"""
from typing import Dict, Optional
import trafaret as T
from aiohttp.web import Application
from pydantic import BaseSettings
from servicelib.application_keys import APP_CONFIG_KEY
from socketio import AsyncServer
CONFIG_SECTION_NAME = "socketio"
APP_CLIENT_SOCKET_SERVER_KEY = __name__ + ".socketio_socketio"
APP_CLIENT_SOCKET_DECORATED_HANDLERS_KEY = __name__ + ".socketio_handlers"
schema = T.Dict(
{
T.Key("enabled", default=True, optional=True): T.Or(T.Bool(), T.ToInt()),
}
)
class SocketIOSettings(BaseSettings):
enabled: Optional[bool] = True
def get_config(app: Application) -> Dict:
return app[APP_CONFIG_KEY][CONFIG_SECTION_NAME]
def get_socket_server(app: Application) -> AsyncServer:
return app[APP_CLIENT_SOCKET_SERVER_KEY]
'request_status':
params['request_status'],
'traceback':
params['traceback'],
})
result = await conn.execute(query)
assert result.rowcount == 1
return web.json_response(resp)
@auth_required
@server_status_required(READ_ALLOWED)
@check_api_params(
t.Dict({
t.Key('mark_read', default=False): t.ToBool(),
t.Key('page_size', default=20): t.ToInt(lt=101),
t.Key('page_no', default=1): t.ToInt()
}), )
async def list_logs(request: web.Request, params: Any) -> web.Response:
resp: MutableMapping[str, Any] = {'logs': []}
dbpool = request.app['dbpool']
domain_name = request['user']['domain_name']
user_role = request['user']['role']
user_uuid = request['user']['uuid']
requester_access_key, owner_access_key = await get_access_key_scopes(
request, params)
log.info(
'LIST (ak:{0}/{1})', requester_access_key,
owner_access_key if owner_access_key != requester_access_key else '*')
async with dbpool.acquire() as conn:
"""
from os import environ as env
import logging
import trafaret as T
log = logging.getLogger(__name__)
CONFIG_SCHEMA = T.Dict({
"endpoint": T.String(),
"access_key": T.String(),
"secret_key": T.String(),
"bucket_name": T.String(),
T.Key("secure", default=0): T.ToInt(),
})
# TODO: deprecate!
class Config():
def __init__(self):
# TODO: uniform config classes . see server.config file
S3_ENDPOINT = env.get("S3_ENDPOINT", "minio:9000")
S3_ACCESS_KEY = env.get("S3_ACCESS_KEY", "12345678")
S3_SECRET_KEY = env.get("S3_SECRET_KEY", "12345678")
S3_BUCKET_NAME = env.get("S3_BUCKET_NAME", "simcore")
S3_SECURE = env.get("S3_SECURE", "0")
self._endpoint = S3_ENDPOINT
self._access_key = S3_ACCESS_KEY
""" storage subsystem's configuration
- config-file schema
- settings
"""
import trafaret as T
CONFIG_SECTION_NAME = "storage"
schema = T.Dict({
T.Key("enabled", default=True, optional=True): T.Bool(),
T.Key("host", default="storage"): T.String(),
T.Key("port", default=11111): T.ToInt(),
T.Key("version", default="v0"):
T.Regexp(regexp=r"^v\d+"), # storage API version basepath
})
import trafaret as T
from servicelib.config_schema_utils import addon_section
from servicelib.tracing import schema as tracing_schema
from simcore_sdk.config import db, s3
from . import rest_config
runs_in_container = "SC_BUILD_TARGET" in os.environ
app_schema = T.Dict(
{
T.Key(
"host", default="0.0.0.0" if runs_in_container else "127.0.0.1" # nosec
): T.IP,
"port": T.ToInt(),
"log_level": T.Enum(
"DEBUG", "WARNING", "INFO", "ERROR", "CRITICAL", "FATAL", "NOTSET"
),
"testing": T.Bool(),
T.Key("max_workers", default=8, optional=True): T.ToInt(),
T.Key("monitoring_enabled", default=False): T.Or(
T.Bool(), T.ToInt
), # Int added to use environs
T.Key("test_datcore", optional=True): T.Dict(
{"token_key": T.String(), "token_secret": T.String()}
),
T.Key("disable_services", default=[], optional=True): T.List(T.String()),
}
)
def test_extract_error(self):
result = t.extract_error(t.ToInt(), '')
assert result == 'value can\'t be converted to int'
""" email's subsystem's configuration
- config-file schema
- settings
"""
import trafaret as T
CONFIG_SECTION_NAME = "smtp"
schema = T.Dict(
{
T.Key(
"sender", default="OSPARC support <*****@*****.**>"
): T.String(), # FIXME: email format
"host": T.String(),
"port": T.ToInt(),
T.Key("tls", default=False): T.Or(T.Bool(), T.ToInt),
T.Key("username", default=None): T.Or(T.String, T.Null),
T.Key("password", default=None): T.Or(T.String, T.Null),
}
)
""" socketio subsystem's configuration
- config-file schema
- settings
"""
from typing import Dict
import trafaret as T
from aiohttp import web
from servicelib.application_keys import APP_CONFIG_KEY
from socketio import AsyncServer
CONFIG_SECTION_NAME = "socketio"
APP_CLIENT_SOCKET_SERVER_KEY = __name__ + ".socketio_socketio"
APP_CLIENT_SOCKET_DECORATED_HANDLERS_KEY = __name__ + ".socketio_handlers"
schema = T.Dict({
T.Key("enabled", default=True, optional=True):
T.Or(T.Bool(), T.ToInt()),
})
def get_config(app: web.Application) -> Dict:
return app[APP_CONFIG_KEY][CONFIG_SECTION_NAME]
def get_socket_server(app: web.Application) -> AsyncServer:
return app[APP_CLIENT_SOCKET_SERVER_KEY]
def test_repr(self):
assert repr(t.ToInt()) == '<ToInt>'
import trafaret as t
from citizens_dwh_api.constants import DATE_FORMAT
def _build_trafaret(schema, optional=False):
return t.Dict(
{t.Key(key, optional=optional): item
for key, item in schema.items()})
OptionalCitizenSchema = {
"town": t.String(min_length=1),
"street": t.String(min_length=1),
"building": t.String(min_length=1),
"apartment": t.ToInt(gte=0),
"name": t.String(min_length=1),
"birth_date": t.ToDateTime(
format=DATE_FORMAT # datetime, because pymongo cannot insert date
),
"gender": t.Enum("male", "female"),
"relatives": t.List(t.ToInt(gte=0)),
}
CitizenSchema = {**OptionalCitizenSchema, **{"citizen_id": t.ToInt(gte=0)}}
OptionalCitizen = _build_trafaret(OptionalCitizenSchema, optional=True)
Citizen = _build_trafaret(CitizenSchema, optional=False)
from models_library.settings.redis import RedisConfig
from servicelib.application_keys import APP_CONFIG_KEY
CONFIG_SECTION_NAME = "resource_manager"
APP_CLIENT_REDIS_CLIENT_KEY = __name__ + ".resource_manager.redis_client"
APP_CLIENT_REDIS_LOCK_KEY = __name__ + ".resource_manager.redis_lock"
APP_CLIENT_SOCKET_REGISTRY_KEY = __name__ + ".resource_manager.registry"
APP_RESOURCE_MANAGER_TASKS_KEY = __name__ + ".resource_manager.tasks.key"
APP_GARBAGE_COLLECTOR_KEY = __name__ + ".resource_manager.garbage_collector_key"
# lock names and format strings
GUEST_USER_RC_LOCK_FORMAT = f"{__name__}:redlock:garbage_collect_user:{{user_id}}"
schema = T.Dict({
T.Key("enabled", default=True, optional=True):
T.Or(T.Bool(), T.ToInt()),
T.Key("resource_deletion_timeout_seconds", default=900, optional=True):
T.ToInt(),
T.Key("garbage_collection_interval_seconds", default=30, optional=True):
T.ToInt(),
T.Key("redis", optional=False):
T.Dict({
T.Key("enabled", default=True, optional=True): T.Bool(),
T.Key("host", default="redis", optional=True): T.String(),
T.Key("port", default=6793, optional=True): T.ToInt(),
}),
})
class RedisSection(RedisConfig):
enabled: bool = True
from pydantic import BaseSettings, Field
from models_library.basic_types import PortInt, VersionTag
from servicelib.application_keys import APP_CLIENT_SESSION_KEY, APP_CONFIG_KEY
CONFIG_SECTION_NAME = "catalog"
_default_values = {
"host": os.environ.get("CATALOG_HOST", "catalog"),
"port": int(os.environ.get("CATALOG_PORT", 8000)),
}
schema = T.Dict({
T.Key("enabled", default=True, optional=True): T.Bool(),
T.Key("host", default=_default_values["host"]): T.String(),
T.Key("port", default=_default_values["port"]): T.ToInt(),
T.Key("version", default="v0"):
T.Regexp(regexp=r"^v\d+"), # catalog API version basepath
})
class CatalogSettings(BaseSettings):
enabled: bool = True
host: str = "catalog"
port: PortInt = 8000
vtag: VersionTag = Field("v0",
alias="version",
description="Catalog service API's version tag")
class Config:
prefix = "CATALOG_"