def check_controller_fabric(personality, fabric):
"""
Check controller Fabric configuration override (which essentially is only
for debugging purposes or for people running Crossbar.io FX Service on-premise)
:param fabric: The Fabric configuration to check.
:type fabric: dict
"""
if not isinstance(fabric, Mapping):
raise checkconfig.InvalidConfigException(
"'fabric' in controller configuration must be a dictionary ({} encountered)\n\n"
.format(type(fabric)))
for k in fabric:
if k not in ['transport', 'heartbeat']:
raise checkconfig.InvalidConfigException(
"encountered unknown attribute '{}' in 'fabric' in controller configuration"
.format(k))
if 'transport' in fabric:
checkconfig.check_connecting_transport(personality,
fabric['transport'])
if 'heartbeat' in fabric:
heartbeat = fabric['heartbeat']
checkconfig.check_dict_args(
{
'startup_delay': (False, [int, float]),
'heartbeat_period': (False, [int, float]),
'include_system_stats': (False, [bool]),
'send_workers_heartbeats': (False, [bool]),
'aggregate_workers_heartbeats': (False, [bool]),
}, heartbeat,
"heartbeat configuration: {}".format(pformat(heartbeat)))
def test_sequence_list(self):
"""
A Sequence should accept list
"""
checkconfig.check_dict_args({"foo": (True, [Sequence])},
{"foo": ["a", "real", "sequence"]},
"Nice message for the user")
def check(personality, config: Dict[str, Any]):
"""
Checks the configuration item. When errors are found, an
:class:`crossbar.common.checkconfig.InvalidConfigException` is raised.
:param personality: The node personality class.
:param config: The Web service configuration item.
"""
if 'type' not in config:
raise InvalidConfigException("missing mandatory attribute 'type' in Web service configuration")
if config['type'] != 'catalog':
raise InvalidConfigException('unexpected Web service type "{}"'.format(config['type']))
check_dict_args(
{
# ID of webservice (must be unique for the web transport)
'id': (False, [str]),
# must be equal to "catalog"
'type': (True, [str]),
# filename (relative to node directory) to FbsRepository file (*.bfbs, *.zip or *.zip.sig)
'filename': (True, [str]),
# path to provide to Werkzeug/Routes (eg "/test" rather than "test")
'path': (False, [str]),
},
config,
'FbsRepository Web service configuration:\n{}'.format(pformat(config)))
def parse(personality, obj, id=None):
"""
Parses a generic object (eg a dict) into a typed
object of this class.
:param obj: The generic object to parse.
:type obj: dict
:returns: Router link configuration
:rtype: :class:`crossbar.edge.worker.rlink.RLinkConfig`
"""
# assert isinstance(personality, Personality)
assert type(obj) == dict
assert id is None or type(id) == str
if id:
obj['id'] = id
check_dict_args(
{
'id': (False, [str]),
'realm': (True, [str]),
'transport': (True, [Mapping]),
'authid': (False, [str]),
'exclude_authid': (False, [Sequence]),
'forward_local_events': (False, [bool]),
'forward_remote_events': (False, [bool]),
'forward_local_invocations': (False, [bool]),
'forward_remote_invocations': (False, [bool]),
}, obj, 'router link configuration')
realm = obj['realm']
authid = obj.get('authid', None)
exclude_authid = obj.get('exclude_authid', [])
for aid in exclude_authid:
assert type(aid) == str
forward_local_events = obj.get('forward_local_events', True)
forward_remote_events = obj.get('forward_remote_events', True)
forward_local_invocations = obj.get('forward_local_invocations', True)
forward_remote_invocations = obj.get('forward_remote_invocations',
True)
transport = obj['transport']
check_realm_name(realm)
check_connecting_transport(personality, transport)
config = RLinkConfig(
realm=realm,
transport=transport,
authid=authid,
exclude_authid=exclude_authid,
forward_local_events=forward_local_events,
forward_remote_events=forward_remote_events,
forward_local_invocations=forward_local_invocations,
forward_remote_invocations=forward_remote_invocations,
)
return config
def test_sequence_list(self):
"""
A Sequence should accept list
"""
checkconfig.check_dict_args(
{"foo": (True, [Sequence])},
{"foo": ["a", "real", "sequence"]},
"Nice message for the user"
)
def check(personality, config):
"""
Checks the configuration item. When errors are found, an
InvalidConfigException exception is raised.
:param personality: The node personality class.
:param config: The Web service configuration item.
:raises: crossbar.common.checkconfig.InvalidConfigException
"""
if 'type' not in config:
raise InvalidConfigException(
"missing mandatory attribute 'type' in Web service configuration"
)
if config['type'] != 'archive':
raise InvalidConfigException(
'unexpected Web service type "{}"'.format(config['type']))
check_dict_args(
{
# ID of webservice (must be unique for the web transport)
'id': (False, [str]),
# must be equal to "archive"
'type': (True, [six.text_type]),
# local path to archive file (relative to node directory)
'archive': (True, [six.text_type]),
# download URL for achive to auto-fetch
'origin': (False, [six.text_type]),
# flag to control automatic downloading from origin
'download': (False, [bool]),
# cache archive contents in memory
'cache': (False, [bool]),
# default filename in archive when fetched URL is "" or "/"
'default_object': (False, [six.text_type]),
# archive object prefix: this is prefixed to the path before looking within the archive file
'object_prefix': (False, [six.text_type]),
# configure additional MIME types, sending correct HTTP response headers
'mime_types': (False, [Mapping]),
# list of SHA3-256 hashes (HEX string) the archive file is to be verified against
'hashes': (False, [Sequence]),
# FIXME
'options': (False, [Mapping]),
},
config,
"Static Web from Archive service configuration: {}".format(config))
def test_notDict(self):
"""
A non-dict passed in as the config will raise a
L{checkconfig.InvalidConfigException}.
"""
with self.assertRaises(checkconfig.InvalidConfigException) as e:
checkconfig.check_dict_args({}, [], "msghere")
self.assertEqual("msghere - invalid type for configuration item - expected dict, got list",
str(e.exception))
def test_notDict(self):
"""
A non-dict passed in as the config will raise a
L{checkconfig.InvalidConfigException}.
"""
with self.assertRaises(checkconfig.InvalidConfigException) as e:
checkconfig.check_dict_args({}, [], "msghere")
self.assertEqual("msghere - invalid type for configuration item - expected dict, got list",
str(e.exception))
def test_wrongType(self):
"""
The wrong type (as defined in the spec) passed in the config will raise
a L{checkconfig.InvalidConfigException}.
"""
with self.assertRaises(checkconfig.InvalidConfigException) as e:
checkconfig.check_dict_args({"foo": (False, [list, set])},
{"foo": {}}, "msghere")
self.assertEqual(("msghere - invalid type dict encountered for "
"attribute 'foo', must be one of (list, set)"),
str(e.exception))
def test_wrongType(self):
"""
The wrong type (as defined in the spec) passed in the config will raise
a L{checkconfig.InvalidConfigException}.
"""
with self.assertRaises(checkconfig.InvalidConfigException) as e:
checkconfig.check_dict_args({"foo": (False, [list, set])},
{"foo": {}}, "msghere")
self.assertEqual(("msghere - invalid type dict encountered for "
"attribute 'foo', must be one of (list, set)"),
str(e.exception))
def check(self, config):
"""
Check submonitor configuration item.
Override in your derived submonitor class.
Raise a `crossbar.common.checkconfig.InvalidConfigException` exception
when you find an error in the item configuration.
:param config: The submonitor configuration item to check.
:type config: dict
"""
check_dict_args({}, config, '{} monitor configuration'.format(self.ID))
def check(self, config):
"""
Check submonitor configuration item.
Override in your derived submonitor class.
Raise a `crossbar.common.checkconfig.InvalidConfigException` exception
when you find an error in the item configuration.
:param config: The submonitor configuration item to check.
:type config: dict
"""
check_dict_args({}, config, '{} monitor configuration'.format(self.ID))
def test_sequence_string(self):
"""
A Sequence should not imply we accept strings
"""
with self.assertRaises(checkconfig.InvalidConfigException) as e:
checkconfig.check_dict_args({"foo": (True, [Sequence])},
{"foo": "not really a Sequence"},
"Nice message for the user")
self.assertEqual(
"Nice message for the user - invalid type str encountered for "
"attribute 'foo', must be one of (Sequence)",
str(e.exception),
)
def test_sequence_string(self):
"""
A Sequence should not imply we accept strings
"""
with self.assertRaises(checkconfig.InvalidConfigException) as e:
checkconfig.check_dict_args(
{"foo": (True, [Sequence])},
{"foo": "not really a Sequence"},
"Nice message for the user"
)
self.assertEqual(
"Nice message for the user - invalid type str encountered for "
"attribute 'foo', must be one of (Sequence)",
str(e.exception),
)
def check_database(personality, database):
checkconfig.check_dict_args(
{
'type': (True, [six.text_type]),
'path': (True, [six.text_type]),
'maxsize': (False, six.integer_types),
}, database, "database configuration")
if database['type'] not in ['cfxdb']:
raise checkconfig.InvalidConfigException(
'invalid type "{}" in database configuration'.format(
database['type']))
if 'maxsize' in database:
# maxsize must be between 1MB and 1TB
if database['maxsize'] < 2**20 or database['maxsize'] > 2**40:
raise checkconfig.InvalidConfigException(
'invalid maxsize {} in database configuration - must be between 1MB and 1TB'
.format(database['maxsize']))
def check_connection(personality, connection, ignore=[]):
"""
Check a connection item (such as a PostgreSQL or Oracle database connection pool).
"""
if 'id' in connection:
checkconfig.check_id(connection['id'])
if 'type' not in connection:
raise checkconfig.InvalidConfigException(
"missing mandatory attribute 'type' in connection configuration"
)
valid_types = ['postgres']
if connection['type'] not in valid_types:
raise checkconfig.InvalidConfigException(
"invalid type '{}' for connection type - must be one of {}".
format(connection['type'], valid_types))
if connection['type'] == 'postgres':
checkconfig.check_dict_args(
{
'id': (False, [six.text_type]),
'type': (True, [six.text_type]),
'host': (False, [six.text_type]),
'port': (False, six.integer_types),
'database': (True, [six.text_type]),
'user': (True, [six.text_type]),
'password': (False, [six.text_type]),
'options': (False, [Mapping]),
}, connection, "PostgreSQL connection configuration")
if 'port' in connection:
checkconfig.check_endpoint_port(connection['port'])
if 'options' in connection:
checkconfig.check_dict_args(
{
'min_connections': (False, six.integer_types),
'max_connections': (False, six.integer_types),
}, connection['options'], "PostgreSQL connection options")
else:
raise checkconfig.InvalidConfigException('logic error')
def check_market_maker(personality, maker):
maker = dict(maker)
checkconfig.check_dict_args(
{
'id': (True, [six.text_type]),
'key': (True, [six.text_type]),
'database': (True, [Mapping]),
'connection': (True, [Mapping]),
'blockchain': (False, [Mapping]),
}, maker, "market maker configuration {}".format(pformat(maker)))
check_database(personality, dict(maker['database']))
checkconfig.check_dict_args(
{
'realm': (True, [six.text_type]),
'transport': (True, [Mapping]),
}, dict(maker['connection']), "market maker connection configuration")
checkconfig.check_connecting_transport(
personality, dict(maker['connection']['transport']))
if 'blockchain' in maker:
check_blockchain(personality, maker['blockchain'])
def check_blockchain(personality, blockchain):
# Examples:
#
# "blockchain": {
# "type": "ethereum",
# "gateway": {
# "type": "auto"
# }
# }
#
# "blockchain": {
# "type": "ethereum",
# "gateway": {
# "type": "user",
# "http": "http://127.0.0.1:8545"
# "websocket": "ws://127.0.0.1:8545"
# },
# "from_block": 1,
# "chain_id": 5777
# }
#
# "blockchain": {
# "type": "ethereum",
# "gateway": {
# "type": "infura",
# "network": "ropsten",
# "key": "00000000000000000000000000000000",
# "secret": "00000000000000000000000000000000"
# },
# "from_block": 6350652,
# "chain_id": 3
# }
checkconfig.check_dict_args(
{
'id': (False, [six.text_type]),
'type': (True, [six.text_type]),
'gateway': (True, [Mapping]),
'key': (False, [six.text_type]),
'from_block': (False, [int]),
'chain_id': (False, [int]),
}, blockchain,
"blockchain configuration item {}".format(pformat(blockchain)))
if blockchain['type'] not in ['ethereum']:
raise checkconfig.InvalidConfigException(
'invalid type "{}" in blockchain configuration'.format(
blockchain['type']))
gateway = blockchain['gateway']
if 'type' not in gateway:
raise checkconfig.InvalidConfigException(
'missing type in gateway item "{}" of blockchain configuration'.
format(pformat(gateway)))
if gateway['type'] not in ['infura', 'user', 'auto']:
raise checkconfig.InvalidConfigException(
'invalid type "{}" in gateway item of blockchain configuration'.
format(gateway['type']))
if gateway['type'] == 'infura':
checkconfig.check_dict_args(
{
'type': (True, [six.text_type]),
'network': (True, [six.text_type]),
'key': (True, [six.text_type]),
'secret': (True, [six.text_type]),
}, gateway,
"blockchain gateway configuration {}".format(pformat(gateway)))
# allow to set value from environment variable
gateway['key'] = checkconfig.maybe_from_env(
'blockchain.gateway["infura"].key',
gateway['key'],
hide_value=True)
gateway['secret'] = checkconfig.maybe_from_env(
'blockchain.gateway["infura"].secret',
gateway['secret'],
hide_value=True)
elif gateway['type'] == 'user':
checkconfig.check_dict_args(
{
'type': (True, [six.text_type]),
'http': (True, [six.text_type]),
# 'websocket': (True, [six.text_type]),
},
gateway,
"blockchain gateway configuration {}".format(pformat(gateway)))
elif gateway['type'] == 'auto':
checkconfig.check_dict_args({
'type': (True, [six.text_type]),
}, gateway, "blockchain gateway configuration {}".format(
pformat(gateway)))
else:
# should not arrive here
raise Exception('logic error')
def check_controller_fabric_center(personality, config):
if not isinstance(config, Mapping):
raise checkconfig.InvalidConfigException(
"'fabric-center' in controller configuration must be a dictionary ({} encountered)\n\n"
.format(type(config)))
for k in config:
if k not in ['metering', 'auto_default_mrealm']:
raise checkconfig.InvalidConfigException(
"encountered unknown attribute '{}' in 'fabric-center' in controller configuration"
.format(k))
if 'auto_default_mrealm' in config:
auto_default_mrealm = config['auto_default_mrealm']
checkconfig.check_dict_args(
{
'enabled': (False, [bool]),
'watch_to_pair': (False, [str]),
'watch_to_pair_pattern': (False, [str]),
'write_pairing_file': (False, [bool]),
}, auto_default_mrealm,
"auto_default_mrealm configuration: {}".format(
pformat(auto_default_mrealm)))
if 'metering' in config:
# "metering": {
# "period": 60,
# "submit": {
# "period": 120,
# "url": "http://localhost:7000",
# "timeout": 5,
# "maxerrors": 10
# }
# }
#
# also possible to read the URL from an env var:
#
# "url": "${crossbar_METERING_URL}"
#
metering = config['metering']
checkconfig.check_dict_args(
{
'period': (False, [int, float]),
'submit': (False, [Mapping]),
}, metering,
"metering configuration: {}".format(pformat(metering)))
if 'submit' in metering:
checkconfig.check_dict_args(
{
'period': (False, [int, float]),
'url': (False, [str]),
'timeout': (False, [int, float]),
'maxerrors': (False, [int]),
}, metering['submit'],
"metering submit configuration: {}".format(
pformat(metering['submit'])))
if 'url' in metering['submit']:
# allow to set value from environment variable
metering['submit']['url'] = checkconfig.maybe_from_env(
'metering.submit.url', metering['submit']['url'])
def from_archive(inventory: 'Inventory', filename: str) -> 'Catalog':
"""
:param inventory:
:param filename:
:return:
"""
if not os.path.isfile(filename):
raise RuntimeError(
'cannot open catalog from archive "{}" - path is not a file'.
format(filename))
if not zipfile.is_zipfile(filename):
raise RuntimeError(
'cannot open catalog from archive "{}" - file is not a ZIP file'
.format(filename))
f = zipfile.ZipFile(filename)
if f.testzip() is not None:
raise RuntimeError(
'cannot open catalog from archive "{}" - ZIP file is corrupt'.
format(filename))
if 'catalog.yaml' not in f.namelist():
raise RuntimeError(
'archive does not seem to be a catalog - missing catalog.yaml catalog index'
)
# open, read and parse catalog metadata file
data = f.open('catalog.yaml').read()
obj = yaml.safe_load(data)
# check metadata object
check_dict_args(
{
# mandatory:
'name': (True, [str]),
'schemas': (True, [Sequence]),
# optional:
'version': (False, [str]),
'title': (False, [str]),
'description': (False, [str]),
'author': (False, [str]),
'publisher': (False, [str]),
'license': (False, [str]),
'keywords': (False, [Sequence]),
'homepage': (False, [str]),
'git': (False, [str]),
'theme': (False, [Mapping]),
},
obj,
"WAMP API Catalog {} invalid".format(filename))
schemas = {}
if 'schemas' in obj:
enum_dups = 0
obj_dups = 0
svc_dups = 0
for schema_path in obj['schemas']:
assert type(schema_path
) == str, 'invalid type {} for schema path'.format(
type(schema_path))
assert schema_path in f.namelist(
), 'cannot find schema path "{}" in catalog archive'.format(
schema_path)
with f.open(schema_path) as fd:
# load FlatBuffers schema object
_schema: FbsSchema = FbsSchema.load(
inventory.repo, fd, schema_path)
# add enum types to repository by name
for _enum in _schema.enums.values():
if _enum.name in inventory.repo._enums:
# print('skipping duplicate enum type for name "{}"'.format(_enum.name))
enum_dups += 1
else:
inventory.repo._enums[_enum.name] = _enum
# add object types to repository by name
for _obj in _schema.objs.values():
if _obj.name in inventory.repo._objs:
# print('skipping duplicate object (table/struct) type for name "{}"'.format(_obj.name))
obj_dups += 1
else:
inventory.repo._objs[_obj.name] = _obj
# add service definitions ("APIs") to repository by name
for _svc in _schema.services.values():
if _svc.name in inventory.repo._services:
# print('skipping duplicate service type for name "{}"'.format(_svc.name))
svc_dups += 1
else:
inventory.repo._services[_svc.name] = _svc
# remember schema object by schema path
schemas[schema_path] = _schema
clicense = None
if 'clicense' in obj:
# FIXME: check SPDX license ID vs
# https://raw.githubusercontent.com/spdx/license-list-data/master/json/licenses.json
clicense = obj['clicense']
keywords = None
if 'keywords' in obj:
kw_pat = re.compile(r'^[a-z]{3,20}$')
for kw in obj['keywords']:
assert type(kw) == str, 'invalid type {} for keyword'.format(
type(kw))
assert kw_pat.match(
kw) is not None, 'invalid keyword "{}"'.format(kw)
keywords = obj['keywords']
homepage = None
if 'homepage' in obj:
assert type(
obj['homepage']) == str, 'invalid type {} for homepage'.format(
type(obj['homepage']))
try:
urlparse(obj['homepage'])
except Exception as e:
raise RuntimeError(
'invalid HTTP(S) URL "{}" for homepage ({})'.format(
obj['homepage'], e))
homepage = obj['homepage']
giturl = None
if 'giturl' in obj:
assert type(
obj['git']) == str, 'invalid type {} for giturl'.format(
type(obj['giturl']))
try:
urlparse(obj['giturl'])
except Exception as e:
raise RuntimeError(
'invalid HTTP(S) URL "{}" for giturl ({})'.format(
obj['giturl'], e))
giturl = obj['giturl']
theme = None
if 'theme' in obj:
assert isinstance(obj['theme'], Mapping)
for k in obj['theme']:
if k not in ['background', 'highlight', 'text', 'logo']:
raise RuntimeError(
'invalid theme attribute "{}"'.format(k))
if type(obj['theme'][k]) != str:
raise RuntimeError(
'invalid type{} for attribute {} in theme'.format(
type(obj['theme'][k]), k))
if 'logo' in obj['theme']:
logo_path = obj['theme']['logo']
assert logo_path in f.namelist(
), 'cannot find theme logo path "{}" in catalog archive'.format(
logo_path)
theme = dict(obj['theme'])
# FIXME: check other theme attributes
publisher = None
if 'publisher' in obj:
# FIXME: check publisher address
publisher = obj['publisher']
catalog = Catalog(inventory=inventory,
ctype=Catalog.CATALOG_TYPE_ARCHIVE,
name=obj['name'],
archive=filename,
version=obj.get('version', None),
title=obj.get('title', None),
description=obj.get('description', None),
schemas=schemas,
author=obj.get('author', None),
publisher=publisher,
clicense=clicense,
keywords=keywords,
homepage=homepage,
giturl=giturl,
theme=theme)
return catalog
def check(personality, config):
"""
Checks the configuration item. When errors are found, an
InvalidConfigException exception is raised.
:param personality: The node personality class.
:param config: The Web service configuration item.
:raises: crossbar.common.checkconfig.InvalidConfigException
"""
if 'type' not in config:
raise InvalidConfigException(
"missing mandatory attribute 'type' in Web service configuration"
)
if config['type'] != 'wap':
raise InvalidConfigException(
'unexpected Web service type "{}"'.format(config['type']))
check_dict_args(
{
# ID of webservice (must be unique for the web transport)
'id': (False, [str]),
# must be equal to "wap"
'type': (True, [str]),
# path to prvide to Werkzeug/Routes (eg "/test" rather than "test")
'path': (False, [str]),
# local directory or package+resource
'templates': (True, [str, Mapping]),
# create sandboxed jinja2 environment
'sandbox': (False, [bool]),
# Web routes
'routes': (True, [Sequence]),
# WAMP connection configuration
'wamp': (True, [Mapping]),
},
config,
"WAMP Application Page (WAP) service configuration".format(config))
if isinstance(config['templates'], Mapping):
check_dict_args(
{
'package': (True, [str]),
'resource': (True, [str]),
}, config['templates'],
"templates in WAP service configuration")
for route in config['routes']:
check_dict_args(
{
'path': (True, [str]),
'method': (True, [str]),
'call': (False, [str, type(None)]),
'render': (True, [str]),
}, route, "route in WAP service configuration")
check_dict_args({
'realm': (True, [str]),
'authrole': (True, [str]),
}, config['wamp'], "wamp in WAP service configuration")