def start_from_config(self, config):
controller_config = config.get('controller', {})
controller_options = controller_config.get('options', {})
controller_title = controller_options.get('title',
'crossbar-controller')
try:
import setproctitle
except ImportError:
log.msg(
"Warning, could not set process title (setproctitle not installed)"
)
else:
setproctitle.setproctitle(controller_title)
## the node's name (must be unique within the management realm)
if 'id' in controller_config:
self._node_id = controller_config['id']
else:
self._node_id = socket.gethostname()
## the node's management realm
self._realm = controller_config.get('realm', 'crossbar')
## the node controller singleton WAMP application session
##
#session_config = ComponentConfig(realm = options.realm, extra = options)
self._controller = NodeControllerSession(self)
## router and factory that creates router sessions
##
self._router_factory = RouterFactory(options=wamp.types.RouterOptions(
uri_check=wamp.types.RouterOptions.URI_CHECK_LOOSE),
debug=False)
self._router_session_factory = RouterSessionFactory(
self._router_factory)
## add the node controller singleton session to the router
##
self._router_session_factory.add(self._controller)
## Detect WAMPlets
##
wamplets = self._controller._get_wamplets()
if len(wamplets) > 0:
log.msg("Detected {} WAMPlets in environment:".format(
len(wamplets)))
for wpl in wamplets:
log.msg("WAMPlet {}.{}".format(wpl['dist'], wpl['name']))
else:
log.msg("No WAMPlets detected in enviroment.")
self.run_node_config(config)
def test_kill_after_term(self):
reactor = task.Clock()
NodeControllerSession._cleanup_worker(reactor, self.worker)
# should have sent TERM now
calls = self.worker.proto.transport.method_calls
self.assertTrue(calls[0][0] == "signalProcess")
self.assertTrue(calls[0][1] == ('TERM',))
# skip ahead until our KILL. we loop because we only run one
# timed-out thing on each advance maybe? Anyway it runs
# timeout() only twice if I advance(30) here instead...
for x in range(30):
reactor.advance(1)
calls = self.worker.proto.transport.method_calls
self.assertTrue(calls[1][0] == "signalProcess")
self.assertTrue(calls[1][1] == ("KILL",))
def start_from_config(self, config):
controller_config = config.get('controller', {})
controller_options = controller_config.get('options', {})
controller_title = controller_options.get('title', 'crossbar-controller')
try:
import setproctitle
except ImportError:
log.msg("Warning, could not set process title (setproctitle not installed)")
else:
setproctitle.setproctitle(controller_title)
## the node's name (must be unique within the management realm)
if 'id' in controller_config:
self._node_id = controller_config['id']
else:
self._node_id = socket.gethostname()
## the node's management realm
self._realm = controller_config.get('realm', 'crossbar')
## the node controller singleton WAMP application session
##
#session_config = ComponentConfig(realm = options.realm, extra = options)
self._controller = NodeControllerSession(self)
## router and factory that creates router sessions
##
self._router_factory = RouterFactory(
options = wamp.types.RouterOptions(uri_check = wamp.types.RouterOptions.URI_CHECK_LOOSE),
debug = False)
self._router_session_factory = RouterSessionFactory(self._router_factory)
## add the node controller singleton session to the router
##
self._router_session_factory.add(self._controller)
## Detect WAMPlets
##
wamplets = self._controller._get_wamplets()
if len(wamplets) > 0:
log.msg("Detected {} WAMPlets in environment:".format(len(wamplets)))
for wpl in wamplets:
log.msg("WAMPlet {}.{}".format(wpl['dist'], wpl['name']))
else:
log.msg("No WAMPlets detected in enviroment.")
self.run_node_config(config)
def start(self, cdc_mode=False):
"""
Starts this node. This will start a node controller and then spawn new worker
processes as needed.
"""
if not self._config:
raise Exception("No node configuration loaded")
controller_config = self._config.get('controller', {})
controller_options = controller_config.get('options', {})
# set controller process title
#
try:
import setproctitle
except ImportError:
self.log.warn(
"Warning, could not set process title (setproctitle not installed)"
)
else:
setproctitle.setproctitle(
controller_options.get('title', 'crossbar-controller'))
# the node controller realm
#
self._realm = controller_config.get('realm', 'crossbar')
# the node's name (must be unique within the management realm when running
# in "managed mode")
#
if 'id' in controller_config:
self._node_id = controller_config['id']
self.log.info("Node ID '{node_id}' set from config",
node_id=self._node_id)
elif 'CDC_ID' in os.environ:
self._node_id = u'{}'.format(os.environ['CDC_ID'])
self.log.info(
"Node ID '{node_id}' set from environment variable CDC_ID",
node_id=self._node_id)
else:
self._node_id = u'{}'.format(socket.gethostname())
self.log.info("Node ID '{node_id}' set from hostname",
node_id=self._node_id)
# standalone vs managed mode
#
if 'cdc' in controller_config and controller_config['cdc'].get(
'enabled', False):
self._prepare_node_keys()
cdc_config = controller_config['cdc']
# CDC connecting transport
#
if 'transport' in cdc_config:
transport = cdc_config['transport']
if 'tls' in transport['endpoint']:
hostname = transport['endpoint']['tls']['hostname']
else:
raise Exception(
"TLS activated on CDC connection, but 'hostname' not provided"
)
self.log.warn(
"CDC transport configuration overridden from node config!")
else:
transport = {
"type": u"websocket",
"url": u"wss://devops.crossbario.com/ws",
"endpoint": {
"type": u"tcp",
"host": u"devops.crossbario.com",
"port": 443,
"timeout": 5,
"tls": {
"hostname": u"devops.crossbario.com"
}
}
}
hostname = u'devops.crossbario.com'
# CDC management realm
#
if 'realm' in cdc_config:
realm = cdc_config['realm']
self.log.info("CDC management realm '{realm}' set from config",
realm=realm)
elif 'CDC_REALM' in os.environ:
realm = u"{}".format(os.environ['CDC_REALM']).strip()
self.log.info(
"CDC management realm '{realm}' set from enviroment variable CDC_REALM",
realm=realm)
else:
raise Exception(
"CDC management realm not set - either 'realm' must be set in node configuration, or in CDC_REALM enviroment variable"
)
# CDC authentication credentials (for WAMP-CRA)
#
authid = self._node_id
if 'secret' in cdc_config:
authkey = cdc_config['secret']
self.log.info("CDC authentication secret loaded from config")
elif 'CDC_SECRET' in os.environ:
authkey = u"{}".format(os.environ['CDC_SECRET']).strip()
self.log.info(
"CDC authentication secret loaded from environment variable CDC_SECRET"
)
else:
raise Exception(
"CDC authentication secret not set - either 'secret' must be set in node configuration, or in CDC_SECRET enviroment variable"
)
# extra info forwarded to CDC client session
#
extra = {
'node': self,
'onready': Deferred(),
'onexit': Deferred(),
'authid': authid,
'authkey': authkey
}
runner = ApplicationRunner(
url=transport['url'],
realm=realm,
extra=extra,
ssl=optionsForClientTLS(hostname),
debug=False,
debug_wamp=False,
)
try:
self.log.info("Connecting to CDC at '{url}' ..",
url=transport['url'])
yield runner.run(NodeManagementSession, start_reactor=False)
# wait until we have attached to the uplink CDC
self._manager = yield extra['onready']
except Exception as e:
raise Exception("Could not connect to CDC - {}".format(e))
# in managed mode, a node - by default - only shuts down when explicitly asked to,
# or upon a fatal error in the node controller
self._node_shutdown_triggers = [
checkconfig.NODE_SHUTDOWN_ON_SHUTDOWN_REQUESTED
]
self.log.info(
"Connected to Crossbar.io DevOps Center (CDC)! Your node runs in managed mode."
)
else:
self._manager = None
# in standalone mode, a node - by default - is immediately shutting down whenever
# a worker exits (successfully or with error)
self._node_shutdown_triggers = [
checkconfig.NODE_SHUTDOWN_ON_WORKER_EXIT
]
# allow to override node shutdown triggers
#
if 'shutdown' in controller_options:
self.log.info(
"Overriding default node shutdown triggers with {} from node config"
.format(controller_options['shutdown']))
self._node_shutdown_triggers = controller_options['shutdown']
else:
self.log.info("Using default node shutdown triggers {}".format(
self._node_shutdown_triggers))
# router and factory that creates router sessions
#
self._router_factory = RouterFactory(self._node_id)
self._router_session_factory = RouterSessionFactory(
self._router_factory)
rlm_config = {'name': self._realm}
rlm = RouterRealm(None, rlm_config)
# create a new router for the realm
router = self._router_factory.start_realm(rlm)
# add a router/realm service session
cfg = ComponentConfig(self._realm)
rlm.session = RouterServiceSession(cfg, router)
self._router_session_factory.add(rlm.session, authrole=u'trusted')
if self._manager:
self._bridge_session = NodeManagementBridgeSession(
cfg, self, self._manager)
self._router_session_factory.add(self._bridge_session,
authrole=u'trusted')
else:
self._bridge_session = None
# the node controller singleton WAMP application session
#
self._controller = NodeControllerSession(self)
# add the node controller singleton session to the router
#
self._router_session_factory.add(self._controller, authrole=u'trusted')
# Detect WAMPlets
#
wamplets = self._controller._get_wamplets()
if len(wamplets) > 0:
self.log.info("Detected {wamplets} WAMPlets in environment:",
wamplets=len(wamplets))
for wpl in wamplets:
self.log.info("WAMPlet {dist}.{name}",
dist=wpl['dist'],
name=wpl['name'])
else:
self.log.debug("No WAMPlets detected in enviroment.")
panic = False
try:
yield self._startup(self._config)
except ApplicationError as e:
panic = True
self.log.error("{msg}", msg=e.error_message())
except Exception:
panic = True
traceback.print_exc()
if panic:
try:
self._reactor.stop()
except twisted.internet.error.ReactorNotRunning:
pass
class Node(object):
"""
A Crossbar.io node is the running a controller process and one or multiple
worker processes.
A single Crossbar.io node runs exactly one instance of this class, hence
this class can be considered a system singleton.
"""
log = make_logger()
def __init__(self, cbdir=None, reactor=None):
"""
:param cbdir: The node directory to run from.
:type cbdir: unicode
:param reactor: Reactor to run on.
:type reactor: obj or None
"""
# node directory
self._cbdir = cbdir or u'.'
# reactor we should run on
if reactor is None:
from twisted.internet import reactor
self._reactor = reactor
# the node's name (must be unique within the management realm)
self._node_id = None
# the node's management realm
self._realm = None
# config of this node.
self._config = None
# node controller session (a singleton ApplicationSession embedded
# in the local node router)
self._controller = None
# when run in "managed mode", this will hold the uplink WAMP session
# from the node controller to the mananagement application
self._manager = None
# node shutdown triggers, one or more of checkconfig.NODE_SHUTDOWN_MODES
self._node_shutdown_triggers = [
checkconfig.NODE_SHUTDOWN_ON_WORKER_EXIT
]
def load(self, configfile=None):
"""
Check and load the node configuration (usually, from ".crossbar/config.json")
or load built-in CDC default config.
"""
if configfile:
configpath = os.path.join(self._cbdir, configfile)
self.log.debug("Loading node configuration from '{configpath}' ..",
configpath=configpath)
# the following will read the config, check the config and replace
# environment variable references in configuration values ("${MYVAR}") and
# finally return the parsed configuration object
self._config = checkconfig.check_config_file(configpath,
silence=True)
self.log.info("Node configuration loaded from '{configfile}'",
configfile=configfile)
else:
self._config = {u"controller": {u"cdc": {u"enabled": True}}}
checkconfig.check_config(self._config)
self.log.info(
"Node configuration loaded from built-in CDC config.")
def _prepare_node_keys(self):
from nacl.signing import SigningKey
from nacl.encoding import HexEncoder
# make sure CBDIR/.cdc exists
#
cdc_dir = os.path.join(self._cbdir, '.cdc')
if os.path.isdir(cdc_dir):
pass
elif os.path.exists(cdc_dir):
raise Exception(".cdc exists, but isn't a directory")
else:
os.mkdir(cdc_dir)
self.log.info("CDC directory created")
# load node ID, either from .cdc/node.id or from CDC_NODE_ID
#
def split_nid(nid_s):
nid_c = nid_s.strip().split('@')
if len(nid_c) != 2:
raise Exception(
"illegal node principal '{}' - must follow the form <node id>@<management realm>"
.format(nid_s))
node_id, realm = nid_c
# FIXME: regex check node_id and realm
return node_id, realm
nid_file = os.path.join(cdc_dir, 'node.id')
node_id, realm = None, None
if os.path.isfile(nid_file):
with open(nid_file, 'r') as f:
node_id, realm = split_nid(f.read())
elif os.path.exists(nid_file):
raise Exception("{} exists, but isn't a file".format(nid_file))
else:
if 'CDC_NODE_ID' in os.environ:
node_id, realm = split_nid(os.environ['CDC_NODE_ID'])
else:
raise Exception(
"Neither node ID file {} exists nor CDC_NODE_ID environment variable set"
.format(nid_file))
# Load the node key, either from .cdc/node.key or from CDC_NODE_KEY.
# The node key is a Ed25519 key in either raw format (32 bytes) or in
# hex-encoded form (64 characters).
#
# Actually, what's loaded is not the secret Ed25519 key, but the _seed_
# for that key. Private keys are derived from this 32-byte (256-bit)
# random seed value. It is thus the seed value which is sensitive and
# must be protected.
#
skey_file = os.path.join(cdc_dir, 'node.key')
skey = None
if os.path.isfile(skey_file):
# FIXME: check file permissions are 0600!
# This value is read in here.
skey_len = os.path.getsize(skey_file)
if skey_len in (32, 64):
with open(skey_file, 'r') as f:
skey_seed = f.read()
encoder = None
if skey_len == 64:
encoder = HexEncoder
skey = SigningKey(skey_seed, encoder=encoder)
self.log.info("Existing CDC node key loaded from {skey_file}.",
skey_file=skey_file)
else:
raise Exception(
"invalid node key length {} (key must either be 32 raw bytes or hex encoded 32 bytes, hence 64 byte char length)"
)
elif os.path.exists(skey_file):
raise Exception("{} exists, but isn't a file".format(skey_file))
else:
skey = SigningKey.generate()
skey_seed = skey.encode(encoder=HexEncoder)
with open(skey_file, 'w') as f:
f.write(skey_seed)
# set file mode to read only for owner
# 384 (decimal) == 0600 (octal) - we use that for Py2/3 reasons
os.chmod(skey_file, 384)
self.log.info("New CDC node key {skey_file} generated.",
skey_file=skey_file)
return realm, node_id, skey
@inlineCallbacks
def start(self, cdc_mode=False):
"""
Starts this node. This will start a node controller and then spawn new worker
processes as needed.
"""
if not self._config:
raise Exception("No node configuration loaded")
controller_config = self._config.get('controller', {})
controller_options = controller_config.get('options', {})
# set controller process title
#
try:
import setproctitle
except ImportError:
self.log.warn(
"Warning, could not set process title (setproctitle not installed)"
)
else:
setproctitle.setproctitle(
controller_options.get('title', 'crossbar-controller'))
# the node controller realm
#
self._realm = controller_config.get('realm', 'crossbar')
# the node's name (must be unique within the management realm when running
# in "managed mode")
#
if 'id' in controller_config:
self._node_id = controller_config['id']
self.log.info("Node ID '{node_id}' set from config",
node_id=self._node_id)
elif 'CDC_ID' in os.environ:
self._node_id = u'{}'.format(os.environ['CDC_ID'])
self.log.info(
"Node ID '{node_id}' set from environment variable CDC_ID",
node_id=self._node_id)
else:
self._node_id = u'{}'.format(socket.gethostname())
self.log.info("Node ID '{node_id}' set from hostname",
node_id=self._node_id)
# standalone vs managed mode
#
if 'cdc' in controller_config and controller_config['cdc'].get(
'enabled', False):
self._prepare_node_keys()
cdc_config = controller_config['cdc']
# CDC connecting transport
#
if 'transport' in cdc_config:
transport = cdc_config['transport']
if 'tls' in transport['endpoint']:
hostname = transport['endpoint']['tls']['hostname']
else:
raise Exception(
"TLS activated on CDC connection, but 'hostname' not provided"
)
self.log.warn(
"CDC transport configuration overridden from node config!")
else:
transport = {
"type": u"websocket",
"url": u"wss://devops.crossbario.com/ws",
"endpoint": {
"type": u"tcp",
"host": u"devops.crossbario.com",
"port": 443,
"timeout": 5,
"tls": {
"hostname": u"devops.crossbario.com"
}
}
}
hostname = u'devops.crossbario.com'
# CDC management realm
#
if 'realm' in cdc_config:
realm = cdc_config['realm']
self.log.info("CDC management realm '{realm}' set from config",
realm=realm)
elif 'CDC_REALM' in os.environ:
realm = u"{}".format(os.environ['CDC_REALM']).strip()
self.log.info(
"CDC management realm '{realm}' set from enviroment variable CDC_REALM",
realm=realm)
else:
raise Exception(
"CDC management realm not set - either 'realm' must be set in node configuration, or in CDC_REALM enviroment variable"
)
# CDC authentication credentials (for WAMP-CRA)
#
authid = self._node_id
if 'secret' in cdc_config:
authkey = cdc_config['secret']
self.log.info("CDC authentication secret loaded from config")
elif 'CDC_SECRET' in os.environ:
authkey = u"{}".format(os.environ['CDC_SECRET']).strip()
self.log.info(
"CDC authentication secret loaded from environment variable CDC_SECRET"
)
else:
raise Exception(
"CDC authentication secret not set - either 'secret' must be set in node configuration, or in CDC_SECRET enviroment variable"
)
# extra info forwarded to CDC client session
#
extra = {
'node': self,
'onready': Deferred(),
'onexit': Deferred(),
'authid': authid,
'authkey': authkey
}
runner = ApplicationRunner(
url=transport['url'],
realm=realm,
extra=extra,
ssl=optionsForClientTLS(hostname),
debug=False,
debug_wamp=False,
)
try:
self.log.info("Connecting to CDC at '{url}' ..",
url=transport['url'])
yield runner.run(NodeManagementSession, start_reactor=False)
# wait until we have attached to the uplink CDC
self._manager = yield extra['onready']
except Exception as e:
raise Exception("Could not connect to CDC - {}".format(e))
# in managed mode, a node - by default - only shuts down when explicitly asked to,
# or upon a fatal error in the node controller
self._node_shutdown_triggers = [
checkconfig.NODE_SHUTDOWN_ON_SHUTDOWN_REQUESTED
]
self.log.info(
"Connected to Crossbar.io DevOps Center (CDC)! Your node runs in managed mode."
)
else:
self._manager = None
# in standalone mode, a node - by default - is immediately shutting down whenever
# a worker exits (successfully or with error)
self._node_shutdown_triggers = [
checkconfig.NODE_SHUTDOWN_ON_WORKER_EXIT
]
# allow to override node shutdown triggers
#
if 'shutdown' in controller_options:
self.log.info(
"Overriding default node shutdown triggers with {} from node config"
.format(controller_options['shutdown']))
self._node_shutdown_triggers = controller_options['shutdown']
else:
self.log.info("Using default node shutdown triggers {}".format(
self._node_shutdown_triggers))
# router and factory that creates router sessions
#
self._router_factory = RouterFactory(self._node_id)
self._router_session_factory = RouterSessionFactory(
self._router_factory)
rlm_config = {'name': self._realm}
rlm = RouterRealm(None, rlm_config)
# create a new router for the realm
router = self._router_factory.start_realm(rlm)
# add a router/realm service session
cfg = ComponentConfig(self._realm)
rlm.session = RouterServiceSession(cfg, router)
self._router_session_factory.add(rlm.session, authrole=u'trusted')
if self._manager:
self._bridge_session = NodeManagementBridgeSession(
cfg, self, self._manager)
self._router_session_factory.add(self._bridge_session,
authrole=u'trusted')
else:
self._bridge_session = None
# the node controller singleton WAMP application session
#
self._controller = NodeControllerSession(self)
# add the node controller singleton session to the router
#
self._router_session_factory.add(self._controller, authrole=u'trusted')
# Detect WAMPlets
#
wamplets = self._controller._get_wamplets()
if len(wamplets) > 0:
self.log.info("Detected {wamplets} WAMPlets in environment:",
wamplets=len(wamplets))
for wpl in wamplets:
self.log.info("WAMPlet {dist}.{name}",
dist=wpl['dist'],
name=wpl['name'])
else:
self.log.debug("No WAMPlets detected in enviroment.")
panic = False
try:
yield self._startup(self._config)
except ApplicationError as e:
panic = True
self.log.error("{msg}", msg=e.error_message())
except Exception:
panic = True
traceback.print_exc()
if panic:
try:
self._reactor.stop()
except twisted.internet.error.ReactorNotRunning:
pass
@inlineCallbacks
def _startup(self, config):
# fake call details information when calling into
# remoted procedure locally
#
call_details = CallDetails(caller=0)
controller = config.get('controller', {})
# start Manhole in node controller
#
if 'manhole' in controller:
yield self._controller.start_manhole(controller['manhole'],
details=call_details)
# startup all workers
#
worker_no = 1
call_options = CallOptions(disclose_me=True)
for worker in config.get('workers', []):
# worker ID, type and logname
#
if 'id' in worker:
worker_id = worker.pop('id')
else:
worker_id = 'worker{}'.format(worker_no)
worker_no += 1
worker_type = worker['type']
worker_options = worker.get('options', {})
if worker_type == 'router':
worker_logname = "Router '{}'".format(worker_id)
elif worker_type == 'container':
worker_logname = "Container '{}'".format(worker_id)
elif worker_type == 'websocket-testee':
worker_logname = "WebSocketTestee '{}'".format(worker_id)
elif worker_type == 'guest':
worker_logname = "Guest '{}'".format(worker_id)
else:
raise Exception("logic error")
# router/container
#
if worker_type in ['router', 'container', 'websocket-testee']:
# start a new native worker process ..
#
if worker_type == 'router':
yield self._controller.start_router(worker_id,
worker_options,
details=call_details)
elif worker_type == 'container':
yield self._controller.start_container(
worker_id, worker_options, details=call_details)
elif worker_type == 'websocket-testee':
yield self._controller.start_websocket_testee(
worker_id, worker_options, details=call_details)
else:
raise Exception("logic error")
# setup native worker generic stuff
#
if 'pythonpath' in worker_options:
added_paths = yield self._controller.call(
'crossbar.node.{}.worker.{}.add_pythonpath'.format(
self._node_id, worker_id),
worker_options['pythonpath'],
options=call_options)
self.log.debug("{worker}: PYTHONPATH extended for {paths}",
worker=worker_logname,
paths=added_paths)
if 'cpu_affinity' in worker_options:
new_affinity = yield self._controller.call(
'crossbar.node.{}.worker.{}.set_cpu_affinity'.format(
self._node_id, worker_id),
worker_options['cpu_affinity'],
options=call_options)
self.log.debug("{worker}: CPU affinity set to {affinity}",
worker=worker_logname,
affinity=new_affinity)
if 'manhole' in worker:
yield self._controller.call(
'crossbar.node.{}.worker.{}.start_manhole'.format(
self._node_id, worker_id),
worker['manhole'],
options=call_options)
self.log.debug("{worker}: manhole started",
worker=worker_logname)
# setup router worker
#
if worker_type == 'router':
# start realms on router
#
realm_no = 1
for realm in worker.get('realms', []):
if 'id' in realm:
realm_id = realm.pop('id')
else:
realm_id = 'realm{}'.format(realm_no)
realm_no += 1
# extract schema information from WAMP-flavored Markdown
#
schemas = None
if 'schemas' in realm:
schemas = {}
schema_pat = re.compile(r"```javascript(.*?)```",
re.DOTALL)
cnt_files = 0
cnt_decls = 0
for schema_file in realm.pop('schemas'):
schema_file = os.path.join(
self._cbdir, schema_file)
self.log.info(
"{worker}: processing WAMP-flavored Markdown file {schema_file} for WAMP schema declarations",
worker=worker_logname,
schema_file=schema_file)
with open(schema_file, 'r') as f:
cnt_files += 1
for d in schema_pat.findall(f.read()):
try:
o = json.loads(d)
if isinstance(
o, dict
) and '$schema' in o and o[
'$schema'] == u'http://wamp.ws/schema#':
uri = o['uri']
if uri not in schemas:
schemas[uri] = {}
schemas[uri].update(o)
cnt_decls += 1
except Exception:
self.log.failure(
"{worker}: WARNING - failed to process declaration in {schema_file} - {log_failure.value}",
worker=worker_logname,
schema_file=schema_file)
self.log.info(
"{worker}: processed {cnt_files} files extracting {cnt_decls} schema declarations and {len_schemas} URIs",
worker=worker_logname,
cnt_files=cnt_files,
cnt_decls=cnt_decls,
len_schemas=len(schemas))
enable_trace = realm.get('trace', False)
yield self._controller.call(
'crossbar.node.{}.worker.{}.start_router_realm'.
format(self._node_id, worker_id),
realm_id,
realm,
schemas,
enable_trace=enable_trace,
options=call_options)
self.log.info(
"{worker}: realm '{realm_id}' (named '{realm_name}') started",
worker=worker_logname,
realm_id=realm_id,
realm_name=realm['name'],
enable_trace=enable_trace)
# add roles to realm
#
role_no = 1
for role in realm.get('roles', []):
if 'id' in role:
role_id = role.pop('id')
else:
role_id = 'role{}'.format(role_no)
role_no += 1
yield self._controller.call(
'crossbar.node.{}.worker.{}.start_router_realm_role'
.format(self._node_id, worker_id),
realm_id,
role_id,
role,
options=call_options)
self.log.info(
"{}: role '{}' (named '{}') started on realm '{}'"
.format(worker_logname, role_id, role['name'],
realm_id))
# start uplinks for realm
#
uplink_no = 1
for uplink in realm.get('uplinks', []):
if 'id' in uplink:
uplink_id = uplink.pop('id')
else:
uplink_id = 'uplink{}'.format(uplink_no)
uplink_no += 1
yield self._controller.call(
'crossbar.node.{}.worker.{}.start_router_realm_uplink'
.format(self._node_id, worker_id),
realm_id,
uplink_id,
uplink,
options=call_options)
self.log.info(
"{}: uplink '{}' started on realm '{}'".format(
worker_logname, uplink_id, realm_id))
# start connections (such as PostgreSQL database connection pools)
# to run embedded in the router
#
connection_no = 1
for connection in worker.get('connections', []):
if 'id' in connection:
connection_id = connection.pop('id')
else:
connection_id = 'connection{}'.format(
connection_no)
connection_no += 1
yield self._controller.call(
'crossbar.node.{}.worker.{}.start_connection'.
format(self._node_id, worker_id),
connection_id,
connection,
options=call_options)
self.log.info("{}: connection '{}' started".format(
worker_logname, connection_id))
# start components to run embedded in the router
#
component_no = 1
for component in worker.get('components', []):
if 'id' in component:
component_id = component.pop('id')
else:
component_id = 'component{}'.format(component_no)
component_no += 1
yield self._controller.call(
'crossbar.node.{}.worker.{}.start_router_component'
.format(self._node_id, worker_id),
component_id,
component,
options=call_options)
self.log.info("{}: component '{}' started".format(
worker_logname, component_id))
# start transports on router
#
transport_no = 1
for transport in worker['transports']:
if 'id' in transport:
transport_id = transport.pop('id')
else:
transport_id = 'transport{}'.format(transport_no)
transport_no += 1
yield self._controller.call(
'crossbar.node.{}.worker.{}.start_router_transport'
.format(self._node_id, worker_id),
transport_id,
transport,
options=call_options)
self.log.info("{}: transport '{}' started".format(
worker_logname, transport_id))
# setup container worker
#
elif worker_type == 'container':
component_no = 1
# if components exit "very soon after" we try to
# start them, we consider that a failure and shut
# our node down. We remove this subscription 2
# seconds after we're done starting everything
# (see below). This is necessary as
# start_container_component returns as soon as
# we've established a connection to the component
def component_exited(info):
component_id = info.get("id")
self.log.critical(
"Component '{component_id}' failed to start; shutting down node.",
component_id=component_id)
try:
self._reactor.stop()
except twisted.internet.error.ReactorNotRunning:
pass
topic = 'crossbar.node.{}.worker.{}.container.on_component_stop'.format(
self._node_id, worker_id)
component_stop_sub = yield self._controller.subscribe(
component_exited, topic)
# start connections (such as PostgreSQL database connection pools)
# to run embedded in the container
#
connection_no = 1
for connection in worker.get('connections', []):
if 'id' in connection:
connection_id = connection.pop('id')
else:
connection_id = 'connection{}'.format(
connection_no)
connection_no += 1
yield self._controller.call(
'crossbar.node.{}.worker.{}.start_connection'.
format(self._node_id, worker_id),
connection_id,
connection,
options=call_options)
self.log.info("{}: connection '{}' started".format(
worker_logname, connection_id))
# start components to run embedded in the container
#
for component in worker.get('components', []):
if 'id' in component:
component_id = component.pop('id')
else:
component_id = 'component{}'.format(component_no)
component_no += 1
yield self._controller.call(
'crossbar.node.{}.worker.{}.start_container_component'
.format(self._node_id, worker_id),
component_id,
component,
options=call_options)
self.log.info(
"{worker}: component '{component_id}' started",
worker=worker_logname,
component_id=component_id)
# after 2 seconds, consider all the application components running
self._reactor.callLater(2, component_stop_sub.unsubscribe)
# setup websocket-testee worker
#
elif worker_type == 'websocket-testee':
# start transports on router
#
transport = worker['transport']
transport_no = 1
transport_id = 'transport{}'.format(transport_no)
yield self._controller.call(
'crossbar.node.{}.worker.{}.start_websocket_testee_transport'
.format(self._node_id, worker_id),
transport_id,
transport,
options=call_options)
self.log.info("{}: transport '{}' started".format(
worker_logname, transport_id))
else:
raise Exception("logic error")
elif worker_type == 'guest':
# start guest worker
#
yield self._controller.start_guest(worker_id,
worker,
details=call_details)
self.log.info("{worker}: started", worker=worker_logname)
else:
raise Exception("logic error")
def start(self):
"""
Starts this node. This will start a node controller and then spawn new worker
processes as needed.
"""
if not self._config:
self.check_config()
controller_config = self._config.get('controller', {})
controller_options = controller_config.get('options', {})
controller_title = controller_options.get('title', 'crossbar-controller')
try:
import setproctitle
except ImportError:
self.log.warn("Warning, could not set process title (setproctitle not installed)")
else:
setproctitle.setproctitle(controller_title)
# the node's name (must be unique within the management realm)
if 'id' in controller_config:
self._node_id = controller_config['id']
else:
self._node_id = socket.gethostname()
if 'manager' in controller_config:
extra = {
'onready': Deferred(),
# authentication information for connecting to uplinkg CDC router
# using WAMP-CRA authentication
#
'authid': self._node_id,
'authkey': controller_config['manager']['key']
}
realm = controller_config['manager']['realm']
transport = controller_config['manager']['transport']
runner = ApplicationRunner(url=transport['url'], realm=realm, extra=extra, debug_wamp=False)
runner.run(NodeManagementSession, start_reactor=False)
# wait until we have attached to the uplink CDC
self._management_session = yield extra['onready']
self.log.info("Node is connected to Crossbar.io DevOps Center (CDC)")
else:
self._management_session = None
# the node's management realm
self._realm = controller_config.get('realm', 'crossbar')
# router and factory that creates router sessions
#
self._router_factory = RouterFactory()
self._router_session_factory = RouterSessionFactory(self._router_factory)
rlm_config = {
'name': self._realm
}
rlm = RouterRealm(None, rlm_config)
# create a new router for the realm
router = self._router_factory.start_realm(rlm)
# add a router/realm service session
cfg = ComponentConfig(self._realm)
rlm.session = RouterServiceSession(cfg, router)
self._router_session_factory.add(rlm.session, authrole=u'trusted')
if self._management_session:
self._bridge_session = NodeManagementBridgeSession(cfg, self._management_session)
self._router_session_factory.add(self._bridge_session, authrole=u'trusted')
else:
self._bridge_session = None
# the node controller singleton WAMP application session
#
self._controller = NodeControllerSession(self)
# add the node controller singleton session to the router
#
self._router_session_factory.add(self._controller, authrole=u'trusted')
# Detect WAMPlets
#
wamplets = self._controller._get_wamplets()
if len(wamplets) > 0:
self.log.info("Detected {wamplets} WAMPlets in environment:",
wamplets=len(wamplets))
for wpl in wamplets:
self.log.info("WAMPlet {dist}.{name}",
dist=wpl['dist'], name=wpl['name'])
else:
self.log.info("No WAMPlets detected in enviroment.")
panic = False
try:
yield self._startup(self._config)
except ApplicationError as e:
panic = True
for line in e.args[0].strip().splitlines():
self.log.error(line)
except Exception:
panic = True
traceback.print_exc()
if panic:
try:
self._reactor.stop()
except twisted.internet.error.ReactorNotRunning:
pass
def start(self, cdc_mode=False):
"""
Starts this node. This will start a node controller and then spawn new worker
processes as needed.
"""
if not self._config:
raise Exception("No node configuration loaded")
controller_config = self._config.get('controller', {})
controller_options = controller_config.get('options', {})
# set controller process title
#
try:
import setproctitle
except ImportError:
self.log.warn("Warning, could not set process title (setproctitle not installed)")
else:
setproctitle.setproctitle(controller_options.get('title', 'crossbar-controller'))
# the node controller realm
#
self._realm = controller_config.get(u'realm', u'crossbar')
# the node's name (must be unique within the management realm when running
# in "managed mode")
#
if 'id' in controller_config:
self._node_id = controller_config['id']
self.log.info("Node ID '{node_id}' set from config", node_id=self._node_id)
elif 'CDC_ID' in os.environ:
self._node_id = u'{}'.format(os.environ['CDC_ID'])
self.log.info("Node ID '{node_id}' set from environment variable CDC_ID", node_id=self._node_id)
else:
self._node_id = u'{}'.format(socket.gethostname())
self.log.info("Node ID '{node_id}' set from hostname", node_id=self._node_id)
# standalone vs managed mode
#
if 'cdc' in controller_config and controller_config['cdc'].get('enabled', False):
self._prepare_node_keys()
cdc_config = controller_config['cdc']
# CDC connecting transport
#
if 'transport' in cdc_config:
transport = cdc_config['transport']
if 'tls' in transport['endpoint']:
if 'hostname' in transport['endpoint']:
hostname = transport['endpoint']['tls']['hostname']
else:
raise Exception("TLS activated on CDC connection, but 'hostname' not provided")
else:
hostname = None
self.log.warn("CDC transport configuration overridden from node config!")
else:
transport = {
"type": u"websocket",
"url": u"wss://devops.crossbario.com/ws",
"endpoint": {
"type": u"tcp",
"host": u"devops.crossbario.com",
"port": 443,
"timeout": 5,
"tls": {
"hostname": u"devops.crossbario.com"
}
}
}
hostname = u'devops.crossbario.com'
# CDC management realm
#
if 'realm' in cdc_config:
realm = cdc_config['realm']
self.log.info("CDC management realm '{realm}' set from config", realm=realm)
elif 'CDC_REALM' in os.environ:
realm = u"{}".format(os.environ['CDC_REALM']).strip()
self.log.info("CDC management realm '{realm}' set from enviroment variable CDC_REALM", realm=realm)
else:
raise Exception("CDC management realm not set - either 'realm' must be set in node configuration, or in CDC_REALM enviroment variable")
# CDC authentication credentials (for WAMP-CRA)
#
authid = self._node_id
if 'secret' in cdc_config:
authkey = cdc_config['secret']
self.log.info("CDC authentication secret loaded from config")
elif 'CDC_SECRET' in os.environ:
authkey = u"{}".format(os.environ['CDC_SECRET']).strip()
self.log.info("CDC authentication secret loaded from environment variable CDC_SECRET")
else:
raise Exception("CDC authentication secret not set - either 'secret' must be set in node configuration, or in CDC_SECRET enviroment variable")
# extra info forwarded to CDC client session
#
extra = {
'node': self,
'onready': Deferred(),
'onexit': Deferred(),
'authid': authid,
'authkey': authkey
}
runner = ApplicationRunner(
url=transport['url'], realm=realm, extra=extra,
ssl=optionsForClientTLS(hostname) if hostname else None,
)
try:
self.log.info("Connecting to CDC at '{url}' ..", url=transport['url'])
yield runner.run(NodeManagementSession, start_reactor=False)
# wait until we have attached to the uplink CDC
self._manager = yield extra['onready']
except Exception as e:
raise Exception("Could not connect to CDC - {}".format(e))
# in managed mode, a node - by default - only shuts down when explicitly asked to,
# or upon a fatal error in the node controller
self._node_shutdown_triggers = [checkconfig.NODE_SHUTDOWN_ON_SHUTDOWN_REQUESTED]
self.log.info("Connected to Crossbar.io DevOps Center (CDC)! Your node runs in managed mode.")
else:
self._manager = None
# in standalone mode, a node - by default - is immediately shutting down whenever
# a worker exits (successfully or with error)
self._node_shutdown_triggers = [checkconfig.NODE_SHUTDOWN_ON_WORKER_EXIT]
# allow to override node shutdown triggers
#
if 'shutdown' in controller_options:
self.log.info("Overriding default node shutdown triggers with {} from node config".format(controller_options['shutdown']))
self._node_shutdown_triggers = controller_options['shutdown']
else:
self.log.info("Using default node shutdown triggers {}".format(self._node_shutdown_triggers))
# router and factory that creates router sessions
#
self._router_factory = RouterFactory(self._node_id)
self._router_session_factory = RouterSessionFactory(self._router_factory)
rlm_config = {
'name': self._realm
}
rlm = RouterRealm(None, rlm_config)
# create a new router for the realm
router = self._router_factory.start_realm(rlm)
# add a router/realm service session
cfg = ComponentConfig(self._realm)
rlm.session = RouterServiceSession(cfg, router)
self._router_session_factory.add(rlm.session, authrole=u'trusted')
if self._manager:
self._bridge_session = NodeManagementBridgeSession(cfg, self, self._manager)
self._router_session_factory.add(self._bridge_session, authrole=u'trusted')
else:
self._bridge_session = None
# the node controller singleton WAMP application session
#
self._controller = NodeControllerSession(self)
# add the node controller singleton session to the router
#
self._router_session_factory.add(self._controller, authrole=u'trusted')
# Detect WAMPlets
#
wamplets = self._controller._get_wamplets()
if len(wamplets) > 0:
self.log.info("Detected {wamplets} WAMPlets in environment:",
wamplets=len(wamplets))
for wpl in wamplets:
self.log.info("WAMPlet {dist}.{name}",
dist=wpl['dist'], name=wpl['name'])
else:
self.log.debug("No WAMPlets detected in enviroment.")
panic = False
try:
yield self._startup(self._config)
# Notify systemd that crossbar is fully up and running
# This has no effect on non-systemd platforms
try:
import sdnotify
sdnotify.SystemdNotifier().notify("READY=1")
except:
pass
except ApplicationError as e:
panic = True
self.log.error("{msg}", msg=e.error_message())
except Exception:
panic = True
traceback.print_exc()
if panic:
try:
self._reactor.stop()
except twisted.internet.error.ReactorNotRunning:
pass
class Node(object):
"""
A Crossbar.io node is the running a controller process and one or multiple
worker processes.
A single Crossbar.io node runs exactly one instance of this class, hence
this class can be considered a system singleton.
"""
log = make_logger()
def __init__(self, cbdir=None, reactor=None):
"""
:param cbdir: The node directory to run from.
:type cbdir: unicode
:param reactor: Reactor to run on.
:type reactor: obj or None
"""
# node directory
self._cbdir = cbdir or u'.'
# reactor we should run on
if reactor is None:
from twisted.internet import reactor
self._reactor = reactor
# the node's name (must be unique within the management realm)
self._node_id = None
# the node's management realm
self._realm = None
# config of this node.
self._config = None
# node controller session (a singleton ApplicationSession embedded
# in the local node router)
self._controller = None
# when run in "managed mode", this will hold the uplink WAMP session
# from the node controller to the mananagement application
self._manager = None
# node shutdown triggers, one or more of checkconfig.NODE_SHUTDOWN_MODES
self._node_shutdown_triggers = [checkconfig.NODE_SHUTDOWN_ON_WORKER_EXIT]
# map fro router worker IDs to
self._realm_templates = {}
# for node elements started under specific IDs, and where
# the node configuration does not specify an ID, use a generic
# name numbered sequentially using the counters here
self._worker_no = 1
self._realm_no = 1
self._role_no = 1
self._connection_no = 1
self._transport_no = 1
self._component_no = 1
def load(self, configfile=None):
"""
Check and load the node configuration (usually, from ".crossbar/config.json")
or load built-in CDC default config.
"""
if configfile:
configpath = os.path.join(self._cbdir, configfile)
self.log.debug("Loading node configuration from '{configpath}' ..",
configpath=configpath)
# the following will read the config, check the config and replace
# environment variable references in configuration values ("${MYVAR}") and
# finally return the parsed configuration object
self._config = checkconfig.check_config_file(configpath)
self.log.info("Node configuration loaded from '{configfile}'",
configfile=configfile)
else:
self._config = {
u"controller": {
u"cdc": {
u"enabled": True
}
}
}
checkconfig.check_config(self._config)
self.log.info("Node configuration loaded from built-in CDC config.")
def _prepare_node_keys(self):
from nacl.signing import SigningKey
from nacl.encoding import HexEncoder
# make sure CBDIR/.cdc exists
#
cdc_dir = os.path.join(self._cbdir, '.cdc')
if os.path.isdir(cdc_dir):
pass
elif os.path.exists(cdc_dir):
raise Exception(".cdc exists, but isn't a directory")
else:
os.mkdir(cdc_dir)
self.log.info("CDC directory created")
# load node ID, either from .cdc/node.id or from CDC_NODE_ID
#
def split_nid(nid_s):
nid_c = nid_s.strip().split('@')
if len(nid_c) != 2:
raise Exception("illegal node principal '{}' - must follow the form <node id>@<management realm>".format(nid_s))
node_id, realm = nid_c
# FIXME: regex check node_id and realm
return node_id, realm
nid_file = os.path.join(cdc_dir, 'node.id')
node_id, realm = None, None
if os.path.isfile(nid_file):
with open(nid_file, 'r') as f:
node_id, realm = split_nid(f.read())
elif os.path.exists(nid_file):
raise Exception("{} exists, but isn't a file".format(nid_file))
else:
if 'CDC_NODE_ID' in os.environ:
node_id, realm = split_nid(os.environ['CDC_NODE_ID'])
else:
raise Exception("Neither node ID file {} exists nor CDC_NODE_ID environment variable set".format(nid_file))
# Load the node key, either from .cdc/node.key or from CDC_NODE_KEY.
# The node key is a Ed25519 key in either raw format (32 bytes) or in
# hex-encoded form (64 characters).
#
# Actually, what's loaded is not the secret Ed25519 key, but the _seed_
# for that key. Private keys are derived from this 32-byte (256-bit)
# random seed value. It is thus the seed value which is sensitive and
# must be protected.
#
skey_file = os.path.join(cdc_dir, 'node.key')
skey = None
if os.path.isfile(skey_file):
# FIXME: check file permissions are 0600!
# This value is read in here.
skey_len = os.path.getsize(skey_file)
if skey_len in (32, 64):
with open(skey_file, 'r') as f:
skey_seed = f.read()
encoder = None
if skey_len == 64:
encoder = HexEncoder
skey = SigningKey(skey_seed, encoder=encoder)
self.log.info("Existing CDC node key loaded from {skey_file}.", skey_file=skey_file)
else:
raise Exception("invalid node key length {} (key must either be 32 raw bytes or hex encoded 32 bytes, hence 64 byte char length)")
elif os.path.exists(skey_file):
raise Exception("{} exists, but isn't a file".format(skey_file))
else:
skey = SigningKey.generate()
skey_seed = skey.encode(encoder=HexEncoder)
with open(skey_file, 'w') as f:
f.write(skey_seed)
# set file mode to read only for owner
# 384 (decimal) == 0600 (octal) - we use that for Py2/3 reasons
os.chmod(skey_file, 384)
self.log.info("New CDC node key {skey_file} generated.", skey_file=skey_file)
return realm, node_id, skey
@inlineCallbacks
def start(self, cdc_mode=False):
"""
Starts this node. This will start a node controller and then spawn new worker
processes as needed.
"""
if not self._config:
raise Exception("No node configuration loaded")
controller_config = self._config.get('controller', {})
controller_options = controller_config.get('options', {})
# set controller process title
#
try:
import setproctitle
except ImportError:
self.log.warn("Warning, could not set process title (setproctitle not installed)")
else:
setproctitle.setproctitle(controller_options.get('title', 'crossbar-controller'))
# the node controller realm
#
self._realm = controller_config.get(u'realm', u'crossbar')
# the node's name (must be unique within the management realm when running
# in "managed mode")
#
if 'id' in controller_config:
self._node_id = controller_config['id']
self.log.info("Node ID '{node_id}' set from config", node_id=self._node_id)
elif 'CDC_ID' in os.environ:
self._node_id = u'{}'.format(os.environ['CDC_ID'])
self.log.info("Node ID '{node_id}' set from environment variable CDC_ID", node_id=self._node_id)
else:
self._node_id = u'{}'.format(socket.gethostname())
self.log.info("Node ID '{node_id}' set from hostname", node_id=self._node_id)
# standalone vs managed mode
#
if 'cdc' in controller_config and controller_config['cdc'].get('enabled', False):
self._prepare_node_keys()
cdc_config = controller_config['cdc']
# CDC connecting transport
#
if 'transport' in cdc_config:
transport = cdc_config['transport']
if 'tls' in transport['endpoint']:
if 'hostname' in transport['endpoint']:
hostname = transport['endpoint']['tls']['hostname']
else:
raise Exception("TLS activated on CDC connection, but 'hostname' not provided")
else:
hostname = None
self.log.warn("CDC transport configuration overridden from node config!")
else:
transport = {
"type": u"websocket",
"url": u"wss://devops.crossbario.com/ws",
"endpoint": {
"type": u"tcp",
"host": u"devops.crossbario.com",
"port": 443,
"timeout": 5,
"tls": {
"hostname": u"devops.crossbario.com"
}
}
}
hostname = u'devops.crossbario.com'
# CDC management realm
#
if 'realm' in cdc_config:
realm = cdc_config['realm']
self.log.info("CDC management realm '{realm}' set from config", realm=realm)
elif 'CDC_REALM' in os.environ:
realm = u"{}".format(os.environ['CDC_REALM']).strip()
self.log.info("CDC management realm '{realm}' set from enviroment variable CDC_REALM", realm=realm)
else:
raise Exception("CDC management realm not set - either 'realm' must be set in node configuration, or in CDC_REALM enviroment variable")
# CDC authentication credentials (for WAMP-CRA)
#
authid = self._node_id
if 'secret' in cdc_config:
authkey = cdc_config['secret']
self.log.info("CDC authentication secret loaded from config")
elif 'CDC_SECRET' in os.environ:
authkey = u"{}".format(os.environ['CDC_SECRET']).strip()
self.log.info("CDC authentication secret loaded from environment variable CDC_SECRET")
else:
raise Exception("CDC authentication secret not set - either 'secret' must be set in node configuration, or in CDC_SECRET enviroment variable")
# extra info forwarded to CDC client session
#
extra = {
'node': self,
'onready': Deferred(),
'onexit': Deferred(),
'authid': authid,
'authkey': authkey
}
runner = ApplicationRunner(
url=transport['url'], realm=realm, extra=extra,
ssl=optionsForClientTLS(hostname) if hostname else None,
)
try:
self.log.info("Connecting to CDC at '{url}' ..", url=transport['url'])
yield runner.run(NodeManagementSession, start_reactor=False)
# wait until we have attached to the uplink CDC
self._manager = yield extra['onready']
except Exception as e:
raise Exception("Could not connect to CDC - {}".format(e))
# in managed mode, a node - by default - only shuts down when explicitly asked to,
# or upon a fatal error in the node controller
self._node_shutdown_triggers = [checkconfig.NODE_SHUTDOWN_ON_SHUTDOWN_REQUESTED]
self.log.info("Connected to Crossbar.io DevOps Center (CDC)! Your node runs in managed mode.")
else:
self._manager = None
# in standalone mode, a node - by default - is immediately shutting down whenever
# a worker exits (successfully or with error)
self._node_shutdown_triggers = [checkconfig.NODE_SHUTDOWN_ON_WORKER_EXIT]
# allow to override node shutdown triggers
#
if 'shutdown' in controller_options:
self.log.info("Overriding default node shutdown triggers with {} from node config".format(controller_options['shutdown']))
self._node_shutdown_triggers = controller_options['shutdown']
else:
self.log.info("Using default node shutdown triggers {}".format(self._node_shutdown_triggers))
# router and factory that creates router sessions
#
self._router_factory = RouterFactory(self._node_id)
self._router_session_factory = RouterSessionFactory(self._router_factory)
rlm_config = {
'name': self._realm
}
rlm = RouterRealm(None, rlm_config)
# create a new router for the realm
router = self._router_factory.start_realm(rlm)
# add a router/realm service session
cfg = ComponentConfig(self._realm)
rlm.session = RouterServiceSession(cfg, router)
self._router_session_factory.add(rlm.session, authrole=u'trusted')
if self._manager:
self._bridge_session = NodeManagementBridgeSession(cfg, self, self._manager)
self._router_session_factory.add(self._bridge_session, authrole=u'trusted')
else:
self._bridge_session = None
# the node controller singleton WAMP application session
#
self._controller = NodeControllerSession(self)
# add the node controller singleton session to the router
#
self._router_session_factory.add(self._controller, authrole=u'trusted')
# Detect WAMPlets
#
wamplets = self._controller._get_wamplets()
if len(wamplets) > 0:
self.log.info("Detected {wamplets} WAMPlets in environment:",
wamplets=len(wamplets))
for wpl in wamplets:
self.log.info("WAMPlet {dist}.{name}",
dist=wpl['dist'], name=wpl['name'])
else:
self.log.debug("No WAMPlets detected in enviroment.")
panic = False
try:
yield self._startup(self._config)
# Notify systemd that crossbar is fully up and running
# This has no effect on non-systemd platforms
try:
import sdnotify
sdnotify.SystemdNotifier().notify("READY=1")
except:
pass
except ApplicationError as e:
panic = True
self.log.error("{msg}", msg=e.error_message())
except Exception:
panic = True
traceback.print_exc()
if panic:
try:
self._reactor.stop()
except twisted.internet.error.ReactorNotRunning:
pass
@inlineCallbacks
def _startup(self, config):
"""
Startup elements in the node as specified in the provided node configuration.
"""
# call options we use to call into the local node management API
call_options = CallOptions()
# fake call details we use to call into the local node management API
call_details = CallDetails(caller=0)
# get contoller configuration subpart
controller = config.get('controller', {})
# start Manhole in node controller
if 'manhole' in controller:
yield self._controller.start_manhole(controller['manhole'], details=call_details)
# startup all workers
for worker in config.get('workers', []):
# worker ID
if 'id' in worker:
worker_id = worker.pop('id')
else:
worker_id = 'worker-{:03d}'.format(self._worker_no)
self._worker_no += 1
# worker type - a type of working process from the following fixed list
worker_type = worker['type']
assert(worker_type in ['router', 'container', 'guest', 'websocket-testee'])
# set logname depending on worker type
if worker_type == 'router':
worker_logname = "Router '{}'".format(worker_id)
elif worker_type == 'container':
worker_logname = "Container '{}'".format(worker_id)
elif worker_type == 'websocket-testee':
worker_logname = "WebSocketTestee '{}'".format(worker_id)
elif worker_type == 'guest':
worker_logname = "Guest '{}'".format(worker_id)
else:
raise Exception("logic error")
# any worker specific options
worker_options = worker.get('options', {})
# native worker processes: router, container, websocket-testee
if worker_type in ['router', 'container', 'websocket-testee']:
# start a new native worker process ..
if worker_type == 'router':
yield self._controller.start_router(worker_id, worker_options, details=call_details)
elif worker_type == 'container':
yield self._controller.start_container(worker_id, worker_options, details=call_details)
elif worker_type == 'websocket-testee':
yield self._controller.start_websocket_testee(worker_id, worker_options, details=call_details)
else:
raise Exception("logic error")
# setup native worker generic stuff
if 'pythonpath' in worker_options:
added_paths = yield self._controller.call('crossbar.node.{}.worker.{}.add_pythonpath'.format(self._node_id, worker_id), worker_options['pythonpath'], options=call_options)
self.log.debug("{worker}: PYTHONPATH extended for {paths}",
worker=worker_logname, paths=added_paths)
if 'cpu_affinity' in worker_options:
new_affinity = yield self._controller.call('crossbar.node.{}.worker.{}.set_cpu_affinity'.format(self._node_id, worker_id), worker_options['cpu_affinity'], options=call_options)
self.log.debug("{worker}: CPU affinity set to {affinity}",
worker=worker_logname, affinity=new_affinity)
if 'manhole' in worker:
yield self._controller.call('crossbar.node.{}.worker.{}.start_manhole'.format(self._node_id, worker_id), worker['manhole'], options=call_options)
self.log.debug("{worker}: manhole started",
worker=worker_logname)
# setup router worker
if worker_type == 'router':
# start realms on router
for realm in worker.get('realms', []):
# start realm
if 'id' in realm:
realm_id = realm.pop('id')
else:
realm_id = 'realm-{:03d}'.format(self._realm_no)
self._realm_no += 1
yield self._controller.call('crossbar.node.{}.worker.{}.start_router_realm'.format(self._node_id, worker_id), realm_id, realm, options=call_options)
self.log.info("{worker}: realm '{realm_id}' (named '{realm_name}') started",
worker=worker_logname, realm_id=realm_id, realm_name=realm['name'])
# add roles to realm
for role in realm.get('roles', []):
if 'id' in role:
role_id = role.pop('id')
else:
role_id = 'role-{:03d}'.format(self._role_no)
self._role_no += 1
yield self._controller.call('crossbar.node.{}.worker.{}.start_router_realm_role'.format(self._node_id, worker_id), realm_id, role_id, role, options=call_options)
self.log.info("{}: role '{}' (named '{}') started on realm '{}'".format(worker_logname, role_id, role['name'], realm_id))
# start uplinks for realm
for uplink in realm.get('uplinks', []):
if 'id' in uplink:
uplink_id = uplink.pop('id')
else:
uplink_id = 'uplink-{:03d}'.format(self._uplink_no)
self._uplink_no += 1
yield self._controller.call('crossbar.node.{}.worker.{}.start_router_realm_uplink'.format(self._node_id, worker_id), realm_id, uplink_id, uplink, options=call_options)
self.log.info("{}: uplink '{}' started on realm '{}'".format(worker_logname, uplink_id, realm_id))
# start connections (such as PostgreSQL database connection pools)
# to run embedded in the router
for connection in worker.get('connections', []):
if 'id' in connection:
connection_id = connection.pop('id')
else:
connection_id = 'connection-{:03d}'.format(self._connection_no)
self._connection_no += 1
yield self._controller.call('crossbar.node.{}.worker.{}.start_connection'.format(self._node_id, worker_id), connection_id, connection, options=call_options)
self.log.info("{}: connection '{}' started".format(worker_logname, connection_id))
# start components to run embedded in the router
for component in worker.get('components', []):
if 'id' in component:
component_id = component.pop('id')
else:
component_id = 'component-{:03d}'.format(self._component_no)
self._component_no += 1
yield self._controller.call('crossbar.node.{}.worker.{}.start_router_component'.format(self._node_id, worker_id), component_id, component, options=call_options)
self.log.info("{}: component '{}' started".format(worker_logname, component_id))
# start transports on router
for transport in worker['transports']:
if 'id' in transport:
transport_id = transport.pop('id')
else:
transport_id = 'transport-{:03d}'.format(self._transport_no)
self._transport_no += 1
yield self._controller.call('crossbar.node.{}.worker.{}.start_router_transport'.format(self._node_id, worker_id), transport_id, transport, options=call_options)
self.log.info("{}: transport '{}' started".format(worker_logname, transport_id))
# setup container worker
elif worker_type == 'container':
# if components exit "very soon after" we try to
# start them, we consider that a failure and shut
# our node down. We remove this subscription 2
# seconds after we're done starting everything
# (see below). This is necessary as
# start_container_component returns as soon as
# we've established a connection to the component
def component_exited(info):
component_id = info.get("id")
self.log.critical("Component '{component_id}' failed to start; shutting down node.", component_id=component_id)
try:
self._reactor.stop()
except twisted.internet.error.ReactorNotRunning:
pass
topic = 'crossbar.node.{}.worker.{}.container.on_component_stop'.format(self._node_id, worker_id)
component_stop_sub = yield self._controller.subscribe(component_exited, topic)
# start connections (such as PostgreSQL database connection pools)
# to run embedded in the container
#
for connection in worker.get('connections', []):
if 'id' in connection:
connection_id = connection.pop('id')
else:
connection_id = 'connection-{:03d}'.format(self._connection_no)
self._connection_no += 1
yield self._controller.call('crossbar.node.{}.worker.{}.start_connection'.format(self._node_id, worker_id), connection_id, connection, options=call_options)
self.log.info("{}: connection '{}' started".format(worker_logname, connection_id))
# start components to run embedded in the container
#
for component in worker.get('components', []):
if 'id' in component:
component_id = component.pop('id')
else:
component_id = 'component-{:03d}'.format(self._component_no)
self._component_no += 1
yield self._controller.call('crossbar.node.{}.worker.{}.start_container_component'.format(self._node_id, worker_id), component_id, component, options=call_options)
self.log.info("{worker}: component '{component_id}' started",
worker=worker_logname, component_id=component_id)
# after 2 seconds, consider all the application components running
self._reactor.callLater(2, component_stop_sub.unsubscribe)
# setup websocket-testee worker
elif worker_type == 'websocket-testee':
# start transport on websocket-testee
transport = worker['transport']
transport_id = 'transport-{:03d}'.format(self._transport_no)
self._transport_no = 1
yield self._controller.call('crossbar.node.{}.worker.{}.start_websocket_testee_transport'.format(self._node_id, worker_id), transport_id, transport, options=call_options)
self.log.info("{}: transport '{}' started".format(worker_logname, transport_id))
else:
raise Exception("logic error")
elif worker_type == 'guest':
# start guest worker
#
yield self._controller.start_guest(worker_id, worker, details=call_details)
self.log.info("{worker}: started", worker=worker_logname)
else:
raise Exception("logic error")
class Node(object):
"""
A Crossbar.io node is the running a controller process and one or multiple
worker processes.
A single Crossbar.io node runs exactly one instance of this class, hence
this class can be considered a system singleton.
"""
def __init__(self, reactor, options):
"""
Ctor.
:param reactor: Reactor to run on.
:type reactor: obj
:param options: Options from command line.
:type options: obj
"""
self.log = make_logger()
self.options = options
# the reactor under which we run
self._reactor = reactor
# shortname for reactor to run (when given via explicit option) or None
self._reactor_shortname = options.reactor
# node directory
self._cbdir = options.cbdir
# the node's name (must be unique within the management realm)
self._node_id = None
# the node's management realm
self._realm = None
# node controller session (a singleton ApplicationSession embedded
# in the node's management router)
self._controller = None
@inlineCallbacks
def start(self):
"""
Starts this node. This will start a node controller and then spawn new worker
processes as needed.
"""
# for now, a node is always started from a local configuration
#
configfile = os.path.join(self.options.cbdir, self.options.config)
self.log.info("Starting from node configuration file '{configfile}'",
configfile=configfile)
self._config = check_config_file(configfile, silence=True)
controller_config = self._config.get('controller', {})
controller_options = controller_config.get('options', {})
controller_title = controller_options.get('title', 'crossbar-controller')
try:
import setproctitle
except ImportError:
self.log.warn("Warning, could not set process title (setproctitle not installed)")
else:
setproctitle.setproctitle(controller_title)
# the node's name (must be unique within the management realm)
if 'manager' in self._config:
self._node_id = self._config['manager']['id']
else:
if 'id' in controller_config:
self._node_id = controller_config['id']
else:
self._node_id = socket.gethostname()
if 'manager' in self._config:
extra = {
'onready': Deferred()
}
runner = ApplicationRunner(url=u"ws://localhost:9000", realm=u"cdc-oberstet-1", extra=extra)
runner.run(NodeManagementSession, start_reactor=False)
# wait until we have attached to the uplink CDC
self._management_session = yield extra['onready']
self.log.info("Connected to Crossbar.io Management Cloud: {management_session}",
management_session=self._management_session)
else:
self._management_session = None
# the node's management realm
self._realm = controller_config.get('realm', 'crossbar')
# router and factory that creates router sessions
#
self._router_factory = RouterFactory()
self._router_session_factory = RouterSessionFactory(self._router_factory)
rlm = RouterRealm(None, {'name': self._realm})
# create a new router for the realm
router = self._router_factory.start_realm(rlm)
# add a router/realm service session
cfg = ComponentConfig(self._realm)
rlm.session = RouterServiceSession(cfg, router)
self._router_session_factory.add(rlm.session, authrole=u'trusted')
if self._management_session:
self._bridge_session = NodeManagementBridgeSession(cfg, self._management_session)
self._router_session_factory.add(self._bridge_session, authrole=u'trusted')
else:
self._bridge_session = None
# the node controller singleton WAMP application session
#
# session_config = ComponentConfig(realm = options.realm, extra = options)
self._controller = NodeControllerSession(self)
# add the node controller singleton session to the router
#
self._router_session_factory.add(self._controller, authrole=u'trusted')
# Detect WAMPlets
#
wamplets = self._controller._get_wamplets()
if len(wamplets) > 0:
self.log.info("Detected {wamplets} WAMPlets in environment:",
wamplets=len(wamplets))
for wpl in wamplets:
self.log.info("WAMPlet {dist}.{name}",
dist=wpl['dist'], name=wpl['name'])
else:
self.log.info("No WAMPlets detected in enviroment.")
try:
if 'manager' in self._config:
yield self._startup_managed(self._config)
else:
yield self._startup_standalone(self._config)
except:
traceback.print_exc()
try:
self._reactor.stop()
except twisted.internet.error.ReactorNotRunning:
pass
@inlineCallbacks
def _startup_managed(self, config):
"""
Connect the node to an upstream management application. The node
will run in "managed" mode (as opposed to "standalone" mode).
"""
yield sleep(1)
@inlineCallbacks
def _startup_standalone(self, config):
"""
Setup node according to the local configuration provided. The node
will run in "standalone" mode (as opposed to "managed" mode).
"""
# fake call details information when calling into
# remoted procedure locally
#
call_details = CallDetails(caller=0)
controller = config.get('controller', {})
# start Manhole in node controller
#
if 'manhole' in controller:
yield self._controller.start_manhole(controller['manhole'], details=call_details)
# start local transport for management router
#
if 'transport' in controller:
yield self._controller.start_management_transport(controller['transport'], details=call_details)
# startup all workers
#
worker_no = 1
call_options = CallOptions(disclose_me=True)
for worker in config.get('workers', []):
# worker ID, type and logname
#
if 'id' in worker:
worker_id = worker.pop('id')
else:
worker_id = 'worker{}'.format(worker_no)
worker_no += 1
worker_type = worker['type']
worker_options = worker.get('options', {})
if worker_type == 'router':
worker_logname = "Router '{}'".format(worker_id)
elif worker_type == 'container':
worker_logname = "Container '{}'".format(worker_id)
elif worker_type == 'guest':
worker_logname = "Guest '{}'".format(worker_id)
else:
raise Exception("logic error")
# router/container
#
if worker_type in ['router', 'container']:
# start a new native worker process ..
#
if worker_type == 'router':
yield self._controller.start_router(worker_id, worker_options, details=call_details)
elif worker_type == 'container':
yield self._controller.start_container(worker_id, worker_options, details=call_details)
else:
raise Exception("logic error")
# setup native worker generic stuff
#
if 'pythonpath' in worker_options:
added_paths = yield self._controller.call('crossbar.node.{}.worker.{}.add_pythonpath'.format(self._node_id, worker_id), worker_options['pythonpath'], options=call_options)
self.log.debug("{worker}: PYTHONPATH extended for {paths}",
worker=worker_logname, paths=added_paths)
if 'cpu_affinity' in worker_options:
new_affinity = yield self._controller.call('crossbar.node.{}.worker.{}.set_cpu_affinity'.format(self._node_id, worker_id), worker_options['cpu_affinity'], options=call_options)
self.log.debug("{worker}: CPU affinity set to {affinity}",
worker=worker_logname, affinity=new_affinity)
if 'manhole' in worker:
yield self._controller.call('crossbar.node.{}.worker.{}.start_manhole'.format(self._node_id, worker_id), worker['manhole'], options=call_options)
self.log.debug("{worker}: manhole started",
worker=worker_logname)
# setup router worker
#
if worker_type == 'router':
# start realms on router
#
realm_no = 1
for realm in worker.get('realms', []):
if 'id' in realm:
realm_id = realm.pop('id')
else:
realm_id = 'realm{}'.format(realm_no)
realm_no += 1
# extract schema information from WAMP-flavored Markdown
#
schemas = None
if 'schemas' in realm:
schemas = {}
schema_pat = re.compile(r"```javascript(.*?)```", re.DOTALL)
cnt_files = 0
cnt_decls = 0
for schema_file in realm.pop('schemas'):
schema_file = os.path.join(self.options.cbdir, schema_file)
self.log.info("{worker}: processing WAMP-flavored Markdown file {schema_file} for WAMP schema declarations",
worker=worker_logname, schema_file=schema_file)
with open(schema_file, 'r') as f:
cnt_files += 1
for d in schema_pat.findall(f.read()):
try:
o = json.loads(d)
if isinstance(o, dict) and '$schema' in o and o['$schema'] == u'http://wamp.ws/schema#':
uri = o['uri']
if uri not in schemas:
schemas[uri] = {}
schemas[uri].update(o)
cnt_decls += 1
except Exception:
self.log.failure("{worker}: WARNING - failed to process declaration in {schema_file} - {log_failure.value}",
worker=worker_logname, schema_file=schema_file)
self.log.info("{worker}: processed {cnt_files} files extracting {cnt_decls} schema declarations and {len_schemas} URIs",
worker=worker_logname, cnt_files=cnt_files, cnt_decls=cnt_decls, len_schemas=len(schemas))
yield self._controller.call('crossbar.node.{}.worker.{}.start_router_realm'.format(self._node_id, worker_id), realm_id, realm, schemas, options=call_options)
self.log.info("{worker}: realm '{realm_id}' (named '{realm_name}') started",
worker=worker_logname, realm_id=realm_id, realm_name=realm['name'])
# add roles to realm
#
role_no = 1
for role in realm.get('roles', []):
if 'id' in role:
role_id = role.pop('id')
else:
role_id = 'role{}'.format(role_no)
role_no += 1
yield self._controller.call('crossbar.node.{}.worker.{}.start_router_realm_role'.format(self._node_id, worker_id), realm_id, role_id, role, options=call_options)
self.log.info("{}: role '{}' (named '{}') started on realm '{}'".format(worker_logname, role_id, role['name'], realm_id))
# start components to run embedded in the router
#
component_no = 1
for component in worker.get('components', []):
if 'id' in component:
component_id = component.pop('id')
else:
component_id = 'component{}'.format(component_no)
component_no += 1
yield self._controller.call('crossbar.node.{}.worker.{}.start_router_component'.format(self._node_id, worker_id), component_id, component, options=call_options)
self.log.info("{}: component '{}' started".format(worker_logname, component_id))
# start transports on router
#
transport_no = 1
for transport in worker['transports']:
if 'id' in transport:
transport_id = transport.pop('id')
else:
transport_id = 'transport{}'.format(transport_no)
transport_no += 1
yield self._controller.call('crossbar.node.{}.worker.{}.start_router_transport'.format(self._node_id, worker_id), transport_id, transport, options=call_options)
self.log.info("{}: transport '{}' started".format(worker_logname, transport_id))
# setup container worker
#
elif worker_type == 'container':
component_no = 1
# if components exit "very soon after" we try to
# start them, we consider that a failure and shut
# our node down. We remove this subscription 2
# seconds after we're done starting everything
# (see below). This is necessary as
# start_container_component returns as soon as
# we've established a connection to the component
def component_exited(info):
dead_comp = info['id']
self.log.info("Component '{}' failed to start; shutting down node.".format(dead_comp))
if self._reactor.running:
self._reactor.stop()
topic = 'crossbar.node.{}.worker.{}.container.on_component_stop'.format(self._node_id, worker_id)
component_stop_sub = yield self._controller.subscribe(component_exited, topic)
for component in worker.get('components', []):
if 'id' in component:
component_id = component.pop('id')
else:
component_id = 'component{}'.format(component_no)
component_no += 1
yield self._controller.call('crossbar.node.{}.worker.{}.start_container_component'.format(self._node_id, worker_id), component_id, component, options=call_options)
self.log.info("{worker}: component '{component_id}' started",
worker=worker_logname, component_id=component_id)
# after 2 seconds, consider all the application components running
self._reactor.callLater(2, component_stop_sub.unsubscribe)
else:
raise Exception("logic error")
elif worker_type == 'guest':
# start guest worker
#
yield self._controller.start_guest(worker_id, worker, details=call_details)
self.log.info("{worker}: started", worker=worker_logname)
else:
raise Exception("logic error")
class Node(object):
"""
A Crossbar.io node is the running a controller process and one or multiple
worker processes.
A single Crossbar.io node runs exactly one instance of this class, hence
this class can be considered a system singleton.
"""
log = make_logger()
def __init__(self, cbdir=None, reactor=None):
"""
:param cbdir: The node directory to run from.
:type cbdir: unicode
:param reactor: Reactor to run on.
:type reactor: obj or None
"""
# node directory
self._cbdir = cbdir or u"."
# reactor we should run on
if reactor is None:
from twisted.internet import reactor
self._reactor = reactor
# the node's name (must be unique within the management realm)
self._node_id = None
# the node's management realm
self._realm = None
# config of this node.
self._config = None
# node controller session (a singleton ApplicationSession embedded
# in the local node router)
self._controller = None
# when run in "managed mode", this will hold the uplink WAMP session
# from the node controller to the mananagement application
self._manager = None
# node shutdown triggers, one or more of checkconfig.NODE_SHUTDOWN_MODES
self._node_shutdown_triggers = [checkconfig.NODE_SHUTDOWN_ON_WORKER_EXIT]
def load(self, configfile=None):
"""
Check and load the node configuration (usually, from ".crossbar/config.json")
or load built-in CDC default config.
"""
if configfile:
configpath = os.path.join(self._cbdir, configfile)
self.log.debug("Loading node configuration from '{configpath}' ..", configpath=configpath)
# the following will read the config, check the config and replace
# environment variable references in configuration values ("${MYVAR}") and
# finally return the parsed configuration object
self._config = checkconfig.check_config_file(configpath, silence=True)
self.log.info("Node configuration loaded from '{configfile}'", configfile=configfile)
else:
self._config = {u"controller": {u"cdc": {u"enabled": True}}}
checkconfig.check_config(self._config)
self.log.info("Node configuration loaded from built-in CDC config.")
def _prepare_node_keys(self):
from nacl.signing import SigningKey
from nacl.encoding import HexEncoder
# make sure CBDIR/.cdc exists
#
cdc_dir = os.path.join(self._cbdir, ".cdc")
if os.path.isdir(cdc_dir):
pass
elif os.path.exists(cdc_dir):
raise Exception(".cdc exists, but isn't a directory")
else:
os.mkdir(cdc_dir)
self.log.info("CDC directory created")
# load node ID, either from .cdc/node.id or from CDC_NODE_ID
#
def split_nid(nid_s):
nid_c = nid_s.strip().split("@")
if len(nid_c) != 2:
raise Exception(
"illegal node principal '{}' - must follow the form <node id>@<management realm>".format(nid_s)
)
node_id, realm = nid_c
# FIXME: regex check node_id and realm
return node_id, realm
nid_file = os.path.join(cdc_dir, "node.id")
node_id, realm = None, None
if os.path.isfile(nid_file):
with open(nid_file, "r") as f:
node_id, realm = split_nid(f.read())
elif os.path.exists(nid_file):
raise Exception("{} exists, but isn't a file".format(nid_file))
else:
if "CDC_NODE_ID" in os.environ:
node_id, realm = split_nid(os.environ["CDC_NODE_ID"])
else:
raise Exception(
"Neither node ID file {} exists nor CDC_NODE_ID environment variable set".format(nid_file)
)
# Load the node key, either from .cdc/node.key or from CDC_NODE_KEY.
# The node key is a Ed25519 key in either raw format (32 bytes) or in
# hex-encoded form (64 characters).
#
# Actually, what's loaded is not the secret Ed25519 key, but the _seed_
# for that key. Private keys are derived from this 32-byte (256-bit)
# random seed value. It is thus the seed value which is sensitive and
# must be protected.
#
skey_file = os.path.join(cdc_dir, "node.key")
skey = None
if os.path.isfile(skey_file):
# FIXME: check file permissions are 0600!
# This value is read in here.
skey_len = os.path.getsize(skey_file)
if skey_len in (32, 64):
with open(skey_file, "r") as f:
skey_seed = f.read()
encoder = None
if skey_len == 64:
encoder = HexEncoder
skey = SigningKey(skey_seed, encoder=encoder)
self.log.info("Existing CDC node key loaded from {skey_file}.", skey_file=skey_file)
else:
raise Exception(
"invalid node key length {} (key must either be 32 raw bytes or hex encoded 32 bytes, hence 64 byte char length)"
)
elif os.path.exists(skey_file):
raise Exception("{} exists, but isn't a file".format(skey_file))
else:
skey = SigningKey.generate()
skey_seed = skey.encode(encoder=HexEncoder)
with open(skey_file, "w") as f:
f.write(skey_seed)
# set file mode to read only for owner
# 384 (decimal) == 0600 (octal) - we use that for Py2/3 reasons
os.chmod(skey_file, 384)
self.log.info("New CDC node key {skey_file} generated.", skey_file=skey_file)
return realm, node_id, skey
@inlineCallbacks
def start(self, cdc_mode=False):
"""
Starts this node. This will start a node controller and then spawn new worker
processes as needed.
"""
if not self._config:
raise Exception("No node configuration loaded")
controller_config = self._config.get("controller", {})
controller_options = controller_config.get("options", {})
# set controller process title
#
try:
import setproctitle
except ImportError:
self.log.warn("Warning, could not set process title (setproctitle not installed)")
else:
setproctitle.setproctitle(controller_options.get("title", "crossbar-controller"))
# the node controller realm
#
self._realm = controller_config.get("realm", "crossbar")
# the node's name (must be unique within the management realm when running
# in "managed mode")
#
if "id" in controller_config:
self._node_id = controller_config["id"]
self.log.info("Node ID '{node_id}' set from config", node_id=self._node_id)
elif "CDC_ID" in os.environ:
self._node_id = u"{}".format(os.environ["CDC_ID"])
self.log.info("Node ID '{node_id}' set from environment variable CDC_ID", node_id=self._node_id)
else:
self._node_id = u"{}".format(socket.gethostname())
self.log.info("Node ID '{node_id}' set from hostname", node_id=self._node_id)
# standalone vs managed mode
#
if "cdc" in controller_config and controller_config["cdc"].get("enabled", False):
self._prepare_node_keys()
cdc_config = controller_config["cdc"]
# CDC connecting transport
#
if "transport" in cdc_config:
transport = cdc_config["transport"]
if "tls" in transport["endpoint"]:
hostname = transport["endpoint"]["tls"]["hostname"]
else:
raise Exception("TLS activated on CDC connection, but 'hostname' not provided")
self.log.warn("CDC transport configuration overridden from node config!")
else:
transport = {
"type": u"websocket",
"url": u"wss://devops.crossbario.com/ws",
"endpoint": {
"type": u"tcp",
"host": u"devops.crossbario.com",
"port": 443,
"timeout": 5,
"tls": {"hostname": u"devops.crossbario.com"},
},
}
hostname = u"devops.crossbario.com"
# CDC management realm
#
if "realm" in cdc_config:
realm = cdc_config["realm"]
self.log.info("CDC management realm '{realm}' set from config", realm=realm)
elif "CDC_REALM" in os.environ:
realm = u"{}".format(os.environ["CDC_REALM"]).strip()
self.log.info("CDC management realm '{realm}' set from enviroment variable CDC_REALM", realm=realm)
else:
raise Exception(
"CDC management realm not set - either 'realm' must be set in node configuration, or in CDC_REALM enviroment variable"
)
# CDC authentication credentials (for WAMP-CRA)
#
authid = self._node_id
if "secret" in cdc_config:
authkey = cdc_config["secret"]
self.log.info("CDC authentication secret loaded from config")
elif "CDC_SECRET" in os.environ:
authkey = u"{}".format(os.environ["CDC_SECRET"]).strip()
self.log.info("CDC authentication secret loaded from environment variable CDC_SECRET")
else:
raise Exception(
"CDC authentication secret not set - either 'secret' must be set in node configuration, or in CDC_SECRET enviroment variable"
)
# extra info forwarded to CDC client session
#
extra = {"node": self, "onready": Deferred(), "onexit": Deferred(), "authid": authid, "authkey": authkey}
runner = ApplicationRunner(
url=transport["url"],
realm=realm,
extra=extra,
ssl=optionsForClientTLS(hostname),
debug=False,
debug_wamp=False,
)
try:
self.log.info("Connecting to CDC at '{url}' ..", url=transport["url"])
yield runner.run(NodeManagementSession, start_reactor=False)
# wait until we have attached to the uplink CDC
self._manager = yield extra["onready"]
except Exception as e:
raise Exception("Could not connect to CDC - {}".format(e))
# in managed mode, a node - by default - only shuts down when explicitly asked to,
# or upon a fatal error in the node controller
self._node_shutdown_triggers = [checkconfig.NODE_SHUTDOWN_ON_SHUTDOWN_REQUESTED]
self.log.info("Connected to Crossbar.io DevOps Center (CDC)! Your node runs in managed mode.")
else:
self._manager = None
# in standalone mode, a node - by default - is immediately shutting down whenever
# a worker exits (successfully or with error)
self._node_shutdown_triggers = [checkconfig.NODE_SHUTDOWN_ON_WORKER_EXIT]
# allow to override node shutdown triggers
#
if "shutdown" in controller_options:
self.log.info(
"Overriding default node shutdown triggers with {} from node config".format(
controller_options["shutdown"]
)
)
self._node_shutdown_triggers = controller_options["shutdown"]
else:
self.log.info("Using default node shutdown triggers {}".format(self._node_shutdown_triggers))
# router and factory that creates router sessions
#
self._router_factory = RouterFactory(self._node_id)
self._router_session_factory = RouterSessionFactory(self._router_factory)
rlm_config = {"name": self._realm}
rlm = RouterRealm(None, rlm_config)
# create a new router for the realm
router = self._router_factory.start_realm(rlm)
# add a router/realm service session
cfg = ComponentConfig(self._realm)
rlm.session = RouterServiceSession(cfg, router)
self._router_session_factory.add(rlm.session, authrole=u"trusted")
if self._manager:
self._bridge_session = NodeManagementBridgeSession(cfg, self, self._manager)
self._router_session_factory.add(self._bridge_session, authrole=u"trusted")
else:
self._bridge_session = None
# the node controller singleton WAMP application session
#
self._controller = NodeControllerSession(self)
# add the node controller singleton session to the router
#
self._router_session_factory.add(self._controller, authrole=u"trusted")
# Detect WAMPlets
#
wamplets = self._controller._get_wamplets()
if len(wamplets) > 0:
self.log.info("Detected {wamplets} WAMPlets in environment:", wamplets=len(wamplets))
for wpl in wamplets:
self.log.info("WAMPlet {dist}.{name}", dist=wpl["dist"], name=wpl["name"])
else:
self.log.debug("No WAMPlets detected in enviroment.")
panic = False
try:
yield self._startup(self._config)
except ApplicationError as e:
panic = True
self.log.error("{msg}", msg=e.error_message())
except Exception:
panic = True
traceback.print_exc()
if panic:
try:
self._reactor.stop()
except twisted.internet.error.ReactorNotRunning:
pass
@inlineCallbacks
def _startup(self, config):
# fake call details information when calling into
# remoted procedure locally
#
call_details = CallDetails(caller=0)
controller = config.get("controller", {})
# start Manhole in node controller
#
if "manhole" in controller:
yield self._controller.start_manhole(controller["manhole"], details=call_details)
# startup all workers
#
worker_no = 1
call_options = CallOptions(disclose_me=True)
for worker in config.get("workers", []):
# worker ID, type and logname
#
if "id" in worker:
worker_id = worker.pop("id")
else:
worker_id = "worker{}".format(worker_no)
worker_no += 1
worker_type = worker["type"]
worker_options = worker.get("options", {})
if worker_type == "router":
worker_logname = "Router '{}'".format(worker_id)
elif worker_type == "container":
worker_logname = "Container '{}'".format(worker_id)
elif worker_type == "websocket-testee":
worker_logname = "WebSocketTestee '{}'".format(worker_id)
elif worker_type == "guest":
worker_logname = "Guest '{}'".format(worker_id)
else:
raise Exception("logic error")
# router/container
#
if worker_type in ["router", "container", "websocket-testee"]:
# start a new native worker process ..
#
if worker_type == "router":
yield self._controller.start_router(worker_id, worker_options, details=call_details)
elif worker_type == "container":
yield self._controller.start_container(worker_id, worker_options, details=call_details)
elif worker_type == "websocket-testee":
yield self._controller.start_websocket_testee(worker_id, worker_options, details=call_details)
else:
raise Exception("logic error")
# setup native worker generic stuff
#
if "pythonpath" in worker_options:
added_paths = yield self._controller.call(
"crossbar.node.{}.worker.{}.add_pythonpath".format(self._node_id, worker_id),
worker_options["pythonpath"],
options=call_options,
)
self.log.debug(
"{worker}: PYTHONPATH extended for {paths}", worker=worker_logname, paths=added_paths
)
if "cpu_affinity" in worker_options:
new_affinity = yield self._controller.call(
"crossbar.node.{}.worker.{}.set_cpu_affinity".format(self._node_id, worker_id),
worker_options["cpu_affinity"],
options=call_options,
)
self.log.debug(
"{worker}: CPU affinity set to {affinity}", worker=worker_logname, affinity=new_affinity
)
if "manhole" in worker:
yield self._controller.call(
"crossbar.node.{}.worker.{}.start_manhole".format(self._node_id, worker_id),
worker["manhole"],
options=call_options,
)
self.log.debug("{worker}: manhole started", worker=worker_logname)
# setup router worker
#
if worker_type == "router":
# start realms on router
#
realm_no = 1
for realm in worker.get("realms", []):
if "id" in realm:
realm_id = realm.pop("id")
else:
realm_id = "realm{}".format(realm_no)
realm_no += 1
# extract schema information from WAMP-flavored Markdown
#
schemas = None
if "schemas" in realm:
schemas = {}
schema_pat = re.compile(r"```javascript(.*?)```", re.DOTALL)
cnt_files = 0
cnt_decls = 0
for schema_file in realm.pop("schemas"):
schema_file = os.path.join(self._cbdir, schema_file)
self.log.info(
"{worker}: processing WAMP-flavored Markdown file {schema_file} for WAMP schema declarations",
worker=worker_logname,
schema_file=schema_file,
)
with open(schema_file, "r") as f:
cnt_files += 1
for d in schema_pat.findall(f.read()):
try:
o = json.loads(d)
if (
isinstance(o, dict)
and "$schema" in o
and o["$schema"] == u"http://wamp.ws/schema#"
):
uri = o["uri"]
if uri not in schemas:
schemas[uri] = {}
schemas[uri].update(o)
cnt_decls += 1
except Exception:
self.log.failure(
"{worker}: WARNING - failed to process declaration in {schema_file} - {log_failure.value}",
worker=worker_logname,
schema_file=schema_file,
)
self.log.info(
"{worker}: processed {cnt_files} files extracting {cnt_decls} schema declarations and {len_schemas} URIs",
worker=worker_logname,
cnt_files=cnt_files,
cnt_decls=cnt_decls,
len_schemas=len(schemas),
)
enable_trace = realm.get("trace", False)
yield self._controller.call(
"crossbar.node.{}.worker.{}.start_router_realm".format(self._node_id, worker_id),
realm_id,
realm,
schemas,
enable_trace=enable_trace,
options=call_options,
)
self.log.info(
"{worker}: realm '{realm_id}' (named '{realm_name}') started",
worker=worker_logname,
realm_id=realm_id,
realm_name=realm["name"],
enable_trace=enable_trace,
)
# add roles to realm
#
role_no = 1
for role in realm.get("roles", []):
if "id" in role:
role_id = role.pop("id")
else:
role_id = "role{}".format(role_no)
role_no += 1
yield self._controller.call(
"crossbar.node.{}.worker.{}.start_router_realm_role".format(self._node_id, worker_id),
realm_id,
role_id,
role,
options=call_options,
)
self.log.info(
"{}: role '{}' (named '{}') started on realm '{}'".format(
worker_logname, role_id, role["name"], realm_id
)
)
# start uplinks for realm
#
uplink_no = 1
for uplink in realm.get("uplinks", []):
if "id" in uplink:
uplink_id = uplink.pop("id")
else:
uplink_id = "uplink{}".format(uplink_no)
uplink_no += 1
yield self._controller.call(
"crossbar.node.{}.worker.{}.start_router_realm_uplink".format(self._node_id, worker_id),
realm_id,
uplink_id,
uplink,
options=call_options,
)
self.log.info(
"{}: uplink '{}' started on realm '{}'".format(worker_logname, uplink_id, realm_id)
)
# start connections (such as PostgreSQL database connection pools)
# to run embedded in the router
#
connection_no = 1
for connection in worker.get("connections", []):
if "id" in connection:
connection_id = connection.pop("id")
else:
connection_id = "connection{}".format(connection_no)
connection_no += 1
yield self._controller.call(
"crossbar.node.{}.worker.{}.start_connection".format(self._node_id, worker_id),
connection_id,
connection,
options=call_options,
)
self.log.info("{}: connection '{}' started".format(worker_logname, connection_id))
# start components to run embedded in the router
#
component_no = 1
for component in worker.get("components", []):
if "id" in component:
component_id = component.pop("id")
else:
component_id = "component{}".format(component_no)
component_no += 1
yield self._controller.call(
"crossbar.node.{}.worker.{}.start_router_component".format(self._node_id, worker_id),
component_id,
component,
options=call_options,
)
self.log.info("{}: component '{}' started".format(worker_logname, component_id))
# start transports on router
#
transport_no = 1
for transport in worker["transports"]:
if "id" in transport:
transport_id = transport.pop("id")
else:
transport_id = "transport{}".format(transport_no)
transport_no += 1
yield self._controller.call(
"crossbar.node.{}.worker.{}.start_router_transport".format(self._node_id, worker_id),
transport_id,
transport,
options=call_options,
)
self.log.info("{}: transport '{}' started".format(worker_logname, transport_id))
# setup container worker
#
elif worker_type == "container":
component_no = 1
# if components exit "very soon after" we try to
# start them, we consider that a failure and shut
# our node down. We remove this subscription 2
# seconds after we're done starting everything
# (see below). This is necessary as
# start_container_component returns as soon as
# we've established a connection to the component
def component_exited(info):
component_id = info.get("id")
self.log.critical(
"Component '{component_id}' failed to start; shutting down node.", component_id=component_id
)
try:
self._reactor.stop()
except twisted.internet.error.ReactorNotRunning:
pass
topic = "crossbar.node.{}.worker.{}.container.on_component_stop".format(self._node_id, worker_id)
component_stop_sub = yield self._controller.subscribe(component_exited, topic)
# start connections (such as PostgreSQL database connection pools)
# to run embedded in the container
#
connection_no = 1
for connection in worker.get("connections", []):
if "id" in connection:
connection_id = connection.pop("id")
else:
connection_id = "connection{}".format(connection_no)
connection_no += 1
yield self._controller.call(
"crossbar.node.{}.worker.{}.start_connection".format(self._node_id, worker_id),
connection_id,
connection,
options=call_options,
)
self.log.info("{}: connection '{}' started".format(worker_logname, connection_id))
# start components to run embedded in the container
#
for component in worker.get("components", []):
if "id" in component:
component_id = component.pop("id")
else:
component_id = "component{}".format(component_no)
component_no += 1
yield self._controller.call(
"crossbar.node.{}.worker.{}.start_container_component".format(self._node_id, worker_id),
component_id,
component,
options=call_options,
)
self.log.info(
"{worker}: component '{component_id}' started",
worker=worker_logname,
component_id=component_id,
)
# after 2 seconds, consider all the application components running
self._reactor.callLater(2, component_stop_sub.unsubscribe)
# setup websocket-testee worker
#
elif worker_type == "websocket-testee":
# start transports on router
#
transport = worker["transport"]
transport_no = 1
transport_id = "transport{}".format(transport_no)
yield self._controller.call(
"crossbar.node.{}.worker.{}.start_websocket_testee_transport".format(self._node_id, worker_id),
transport_id,
transport,
options=call_options,
)
self.log.info("{}: transport '{}' started".format(worker_logname, transport_id))
else:
raise Exception("logic error")
elif worker_type == "guest":
# start guest worker
#
yield self._controller.start_guest(worker_id, worker, details=call_details)
self.log.info("{worker}: started", worker=worker_logname)
else:
raise Exception("logic error")
def start(self, cdc_mode=False):
"""
Starts this node. This will start a node controller and then spawn new worker
processes as needed.
"""
if not self._config:
raise Exception("No node configuration loaded")
if not cdc_mode and not self._config.get("controller", {}) and not self._config.get("workers", {}):
self.log.warn(
("You seem to have no controller config or workers, nor are "
"starting up in CDC mode. Check your config exists, or pass "
"--cdc to `crossbar start`."))
try:
self._reactor.stop()
except twisted.internet.error.ReactorNotRunning:
pass
return
# get controller config/options
#
controller_config = self._config.get('controller', {})
controller_options = controller_config.get('options', {})
# set controller process title
#
try:
import setproctitle
except ImportError:
self.log.warn("Warning, could not set process title (setproctitle not installed)")
else:
setproctitle.setproctitle(controller_options.get('title', 'crossbar-controller'))
# router and factory that creates router sessions
#
self._router_factory = RouterFactory()
self._router_session_factory = RouterSessionFactory(self._router_factory)
# create a new router for the realm
#
rlm_config = {
'name': self._realm
}
rlm = RouterRealm(None, rlm_config)
router = self._router_factory.start_realm(rlm)
# always add a realm service session
#
cfg = ComponentConfig(self._realm)
rlm.session = RouterServiceSession(cfg, router)
self._router_session_factory.add(rlm.session, authrole=u'trusted')
# add a router bridge session when running in managed mode
#
if cdc_mode:
self._bridge_session = NodeManagementBridgeSession(cfg)
self._router_session_factory.add(self._bridge_session, authrole=u'trusted')
else:
self._bridge_session = None
# Node shutdown mode
#
if cdc_mode:
# in managed mode, a node - by default - only shuts down when explicitly asked to,
# or upon a fatal error in the node controller
self._node_shutdown_triggers = [checkconfig.NODE_SHUTDOWN_ON_SHUTDOWN_REQUESTED]
else:
# in standalone mode, a node - by default - is immediately shutting down whenever
# a worker exits (successfully or with error)
self._node_shutdown_triggers = [checkconfig.NODE_SHUTDOWN_ON_WORKER_EXIT]
# allow to override node shutdown triggers
#
if 'shutdown' in controller_options:
self.log.info("Overriding default node shutdown triggers with {triggers} from node config", triggers=controller_options['shutdown'])
self._node_shutdown_triggers = controller_options['shutdown']
else:
self.log.info("Using default node shutdown triggers {triggers}", triggers=self._node_shutdown_triggers)
# add the node controller singleton session
#
self._controller = NodeControllerSession(self)
self._router_session_factory.add(self._controller, authrole=u'trusted')
# detect WAMPlets (FIXME: remove this!)
#
wamplets = self._controller._get_wamplets()
if len(wamplets) > 0:
self.log.info("Detected {wamplets} WAMPlets in environment:",
wamplets=len(wamplets))
for wpl in wamplets:
self.log.info("WAMPlet {dist}.{name}",
dist=wpl['dist'], name=wpl['name'])
else:
self.log.debug("No WAMPlets detected in enviroment.")
panic = False
try:
# startup the node from local node configuration
#
yield self._startup(self._config)
# connect to CDC when running in managed mode
#
if cdc_mode:
cdc_config = controller_config.get('cdc', {
# CDC connecting transport
u'transport': {
u'type': u'websocket',
u'url': u'wss://cdc.crossbario.com/ws',
u'endpoint': {
u'type': u'tcp',
u'host': u'cdc.crossbario.com',
u'port': 443,
u'timeout': 5,
u'tls': {
u'hostname': u'cdc.crossbario.com'
}
}
}
})
transport = cdc_config[u'transport']
hostname = None
if u'tls' in transport[u'endpoint']:
transport[u'endpoint'][u'tls'][u'hostname']
runner = ApplicationRunner(
url=transport['url'],
realm=None,
extra=None,
ssl=optionsForClientTLS(hostname) if hostname else None,
)
def make(config):
# extra info forwarded to CDC client session
extra = {
'node': self,
'on_ready': Deferred(),
'on_exit': Deferred(),
'node_key': self._node_key,
}
@inlineCallbacks
def on_ready(res):
self._manager, self._management_realm, self._node_id, self._node_extra = res
if self._bridge_session:
try:
yield self._bridge_session.attach_manager(self._manager, self._management_realm, self._node_id)
status = yield self._manager.call(u'cdc.remote.status@1')
except:
self.log.failure()
else:
self.log.info('Connected to CDC for management realm "{realm}" (current time is {now})', realm=self._management_realm, now=status[u'now'])
else:
self.log.warn('Uplink CDC session established, but no bridge session setup!')
@inlineCallbacks
def on_exit(res):
if self._bridge_session:
try:
yield self._bridge_session.detach_manager()
except:
self.log.failure()
else:
self.log.info('Disconnected from CDC for management realm "{realm}"', realm=self._management_realm)
else:
self.log.warn('Uplink CDC session lost, but no bridge session setup!')
self._manager, self._management_realm, self._node_id, self._node_extra = None, None, None, None
extra['on_ready'].addCallback(on_ready)
extra['on_exit'].addCallback(on_exit)
config = ComponentConfig(extra=extra)
session = NodeManagementSession(config)
return session
self.log.info("Connecting to CDC at '{url}' ..", url=transport[u'url'])
yield runner.run(make, start_reactor=False, auto_reconnect=True)
# Notify systemd that crossbar is fully up and running
# (this has no effect on non-systemd platforms)
try:
import sdnotify
sdnotify.SystemdNotifier().notify("READY=1")
except:
pass
except ApplicationError as e:
panic = True
self.log.error("{msg}", msg=e.error_message())
except Exception:
panic = True
traceback.print_exc()
if panic:
try:
self._reactor.stop()
except twisted.internet.error.ReactorNotRunning:
pass
class Node(object):
"""
A Crossbar.io node is the running a controller process and one or multiple
worker processes.
A single Crossbar.io node runs exactly one instance of this class, hence
this class can be considered a system singleton.
"""
log = make_logger()
def __init__(self, cbdir=None, reactor=None):
"""
:param cbdir: The node directory to run from.
:type cbdir: unicode
:param reactor: Reactor to run on.
:type reactor: obj or None
"""
# node directory
self._cbdir = cbdir or u'.'
# reactor we should run on
if reactor is None:
from twisted.internet import reactor
self._reactor = reactor
# the node's management realm when running in managed mode (this comes from CDC!)
self._management_realm = None
# the node's ID when running in managed mode (this comes from CDC!)
self._node_id = None
# node extra when running in managed mode (this comes from CDC!)
self._node_extra = None
# the node controller realm
self._realm = u'crossbar'
# config of this node.
self._config = None
# node private key autobahn.wamp.cryptosign.SigningKey
self._node_key = None
# node controller session (a singleton ApplicationSession embedded
# in the local node router)
self._controller = None
# when running in managed mode, this will hold the bridge session
# attached to the local management router
self._bridge_session = None
# when running in managed mode, this will hold the uplink session to CDC
self._manager = None
# node shutdown triggers, one or more of checkconfig.NODE_SHUTDOWN_MODES
self._node_shutdown_triggers = [checkconfig.NODE_SHUTDOWN_ON_WORKER_EXIT]
# map from router worker IDs to
self._realm_templates = {}
# for node elements started under specific IDs, and where
# the node configuration does not specify an ID, use a generic
# name numbered sequentially using the counters here
self._worker_no = 1
self._realm_no = 1
self._role_no = 1
self._connection_no = 1
self._transport_no = 1
self._component_no = 1
def maybe_generate_key(self, cbdir, privkey_path=u'key.priv', pubkey_path=u'key.pub'):
privkey_path = os.path.join(cbdir, privkey_path)
pubkey_path = os.path.join(cbdir, pubkey_path)
if os.path.exists(privkey_path):
# node private key seems to exist already .. check!
priv_tags = _parse_keyfile(privkey_path, private=True)
for tag in [u'creator', u'created-at', u'machine-id', u'public-key-ed25519', u'private-key-ed25519']:
if tag not in priv_tags:
raise Exception("Corrupt node private key file {} - {} tag not found".format(privkey_path, tag))
privkey_hex = priv_tags[u'private-key-ed25519']
privkey = SigningKey(privkey_hex, encoder=HexEncoder)
pubkey = privkey.verify_key
pubkey_hex = pubkey.encode(encoder=HexEncoder).decode('ascii')
if priv_tags[u'public-key-ed25519'] != pubkey_hex:
raise Exception(
("Inconsistent node private key file {} - public-key-ed25519 doesn't"
" correspond to private-key-ed25519").format(pubkey_path)
)
if os.path.exists(pubkey_path):
pub_tags = _parse_keyfile(pubkey_path, private=False)
for tag in [u'creator', u'created-at', u'machine-id', u'public-key-ed25519']:
if tag not in pub_tags:
raise Exception("Corrupt node public key file {} - {} tag not found".format(pubkey_path, tag))
if pub_tags[u'public-key-ed25519'] != pubkey_hex:
raise Exception(
("Inconsistent node public key file {} - public-key-ed25519 doesn't"
" correspond to private-key-ed25519").format(pubkey_path)
)
else:
self.log.info(
"Node public key file {pub_path} not found - re-creating from node private key file {priv_path}",
pub_path=pubkey_path,
priv_path=privkey_path,
)
pub_tags = OrderedDict([
(u'creator', priv_tags[u'creator']),
(u'created-at', priv_tags[u'created-at']),
(u'machine-id', priv_tags[u'machine-id']),
(u'public-key-ed25519', pubkey_hex),
])
msg = u'Crossbar.io node public key\n\n'
_write_node_key(pubkey_path, pub_tags, msg)
self.log.debug("Node key already exists (public key: {hex})", hex=pubkey_hex)
else:
# node private key does not yet exist: generate one
privkey = SigningKey.generate()
privkey_hex = privkey.encode(encoder=HexEncoder).decode('ascii')
pubkey = privkey.verify_key
pubkey_hex = pubkey.encode(encoder=HexEncoder).decode('ascii')
# first, write the public file
tags = OrderedDict([
(u'creator', _creator()),
(u'created-at', utcnow()),
(u'machine-id', _machine_id()),
(u'public-key-ed25519', pubkey_hex),
])
msg = u'Crossbar.io node public key\n\n'
_write_node_key(pubkey_path, tags, msg)
# now, add the private key and write the private file
tags[u'private-key-ed25519'] = privkey_hex
msg = u'Crossbar.io node private key - KEEP THIS SAFE!\n\n'
_write_node_key(privkey_path, tags, msg)
self.log.info("New node key pair generated!")
# fix file permissions on node public/private key files
# note: we use decimals instead of octals as octal literals have changed between Py2/3
#
if os.stat(pubkey_path).st_mode & 511 != 420: # 420 (decimal) == 0644 (octal)
os.chmod(pubkey_path, 420)
self.log.info("File permissions on node public key fixed!")
if os.stat(privkey_path).st_mode & 511 != 384: # 384 (decimal) == 0600 (octal)
os.chmod(privkey_path, 384)
self.log.info("File permissions on node private key fixed!")
self._node_key = cryptosign.SigningKey(privkey)
return pubkey_hex
def load(self, configfile=None):
"""
Check and load the node configuration (usually, from ".crossbar/config.json")
or load built-in CDC default config.
"""
if configfile:
configpath = os.path.join(self._cbdir, configfile)
self.log.debug("Loading node configuration from '{configpath}' ..",
configpath=configpath)
# the following will read the config, check the config and replace
# environment variable references in configuration values ("${MYVAR}") and
# finally return the parsed configuration object
self._config = checkconfig.check_config_file(configpath)
self.log.info("Node configuration loaded from '{configfile}'",
configfile=configfile)
else:
self._config = {
u'version': 2,
u'controller': {},
u'workers': []
}
checkconfig.check_config(self._config)
self.log.info("Node configuration loaded from built-in config.")
@inlineCallbacks
def start(self, cdc_mode=False):
"""
Starts this node. This will start a node controller and then spawn new worker
processes as needed.
"""
if not self._config:
raise Exception("No node configuration loaded")
if not cdc_mode and not self._config.get("controller", {}) and not self._config.get("workers", {}):
self.log.warn(
("You seem to have no controller config or workers, nor are "
"starting up in CDC mode. Check your config exists, or pass "
"--cdc to `crossbar start`."))
try:
self._reactor.stop()
except twisted.internet.error.ReactorNotRunning:
pass
return
# get controller config/options
#
controller_config = self._config.get('controller', {})
controller_options = controller_config.get('options', {})
# set controller process title
#
try:
import setproctitle
except ImportError:
self.log.warn("Warning, could not set process title (setproctitle not installed)")
else:
setproctitle.setproctitle(controller_options.get('title', 'crossbar-controller'))
# router and factory that creates router sessions
#
self._router_factory = RouterFactory()
self._router_session_factory = RouterSessionFactory(self._router_factory)
# create a new router for the realm
#
rlm_config = {
'name': self._realm
}
rlm = RouterRealm(None, rlm_config)
router = self._router_factory.start_realm(rlm)
# always add a realm service session
#
cfg = ComponentConfig(self._realm)
rlm.session = RouterServiceSession(cfg, router)
self._router_session_factory.add(rlm.session, authrole=u'trusted')
# add a router bridge session when running in managed mode
#
if cdc_mode:
self._bridge_session = NodeManagementBridgeSession(cfg)
self._router_session_factory.add(self._bridge_session, authrole=u'trusted')
else:
self._bridge_session = None
# Node shutdown mode
#
if cdc_mode:
# in managed mode, a node - by default - only shuts down when explicitly asked to,
# or upon a fatal error in the node controller
self._node_shutdown_triggers = [checkconfig.NODE_SHUTDOWN_ON_SHUTDOWN_REQUESTED]
else:
# in standalone mode, a node - by default - is immediately shutting down whenever
# a worker exits (successfully or with error)
self._node_shutdown_triggers = [checkconfig.NODE_SHUTDOWN_ON_WORKER_EXIT]
# allow to override node shutdown triggers
#
if 'shutdown' in controller_options:
self.log.info("Overriding default node shutdown triggers with {triggers} from node config", triggers=controller_options['shutdown'])
self._node_shutdown_triggers = controller_options['shutdown']
else:
self.log.info("Using default node shutdown triggers {triggers}", triggers=self._node_shutdown_triggers)
# add the node controller singleton session
#
self._controller = NodeControllerSession(self)
self._router_session_factory.add(self._controller, authrole=u'trusted')
# detect WAMPlets (FIXME: remove this!)
#
wamplets = self._controller._get_wamplets()
if len(wamplets) > 0:
self.log.info("Detected {wamplets} WAMPlets in environment:",
wamplets=len(wamplets))
for wpl in wamplets:
self.log.info("WAMPlet {dist}.{name}",
dist=wpl['dist'], name=wpl['name'])
else:
self.log.debug("No WAMPlets detected in enviroment.")
panic = False
try:
# startup the node from local node configuration
#
yield self._startup(self._config)
# connect to CDC when running in managed mode
#
if cdc_mode:
cdc_config = controller_config.get('cdc', {
# CDC connecting transport
u'transport': {
u'type': u'websocket',
u'url': u'wss://cdc.crossbario.com/ws',
u'endpoint': {
u'type': u'tcp',
u'host': u'cdc.crossbario.com',
u'port': 443,
u'timeout': 5,
u'tls': {
u'hostname': u'cdc.crossbario.com'
}
}
}
})
transport = cdc_config[u'transport']
hostname = None
if u'tls' in transport[u'endpoint']:
transport[u'endpoint'][u'tls'][u'hostname']
runner = ApplicationRunner(
url=transport['url'],
realm=None,
extra=None,
ssl=optionsForClientTLS(hostname) if hostname else None,
)
def make(config):
# extra info forwarded to CDC client session
extra = {
'node': self,
'on_ready': Deferred(),
'on_exit': Deferred(),
'node_key': self._node_key,
}
@inlineCallbacks
def on_ready(res):
self._manager, self._management_realm, self._node_id, self._node_extra = res
if self._bridge_session:
try:
yield self._bridge_session.attach_manager(self._manager, self._management_realm, self._node_id)
status = yield self._manager.call(u'cdc.remote.status@1')
except:
self.log.failure()
else:
self.log.info('Connected to CDC for management realm "{realm}" (current time is {now})', realm=self._management_realm, now=status[u'now'])
else:
self.log.warn('Uplink CDC session established, but no bridge session setup!')
@inlineCallbacks
def on_exit(res):
if self._bridge_session:
try:
yield self._bridge_session.detach_manager()
except:
self.log.failure()
else:
self.log.info('Disconnected from CDC for management realm "{realm}"', realm=self._management_realm)
else:
self.log.warn('Uplink CDC session lost, but no bridge session setup!')
self._manager, self._management_realm, self._node_id, self._node_extra = None, None, None, None
extra['on_ready'].addCallback(on_ready)
extra['on_exit'].addCallback(on_exit)
config = ComponentConfig(extra=extra)
session = NodeManagementSession(config)
return session
self.log.info("Connecting to CDC at '{url}' ..", url=transport[u'url'])
yield runner.run(make, start_reactor=False, auto_reconnect=True)
# Notify systemd that crossbar is fully up and running
# (this has no effect on non-systemd platforms)
try:
import sdnotify
sdnotify.SystemdNotifier().notify("READY=1")
except:
pass
except ApplicationError as e:
panic = True
self.log.error("{msg}", msg=e.error_message())
except Exception:
panic = True
traceback.print_exc()
if panic:
try:
self._reactor.stop()
except twisted.internet.error.ReactorNotRunning:
pass
@inlineCallbacks
def _startup(self, config):
"""
Startup elements in the node as specified in the provided node configuration.
"""
self.log.info('Configuring node from local configuration ...')
# call options we use to call into the local node management API
call_options = CallOptions()
# fake call details we use to call into the local node management API
call_details = CallDetails(caller=0)
# get contoller configuration subpart
controller = config.get('controller', {})
# start Manhole in node controller
if 'manhole' in controller:
yield self._controller.start_manhole(controller['manhole'], details=call_details)
# startup all workers
workers = config.get('workers', [])
if len(workers):
self.log.info('Starting {nworkers} workers ...', nworkers=len(workers))
else:
self.log.info('No workers configured!')
for worker in workers:
# worker ID
if 'id' in worker:
worker_id = worker.pop('id')
else:
worker_id = 'worker-{:03d}'.format(self._worker_no)
self._worker_no += 1
# worker type - a type of working process from the following fixed list
worker_type = worker['type']
assert(worker_type in ['router', 'container', 'guest', 'websocket-testee'])
# set logname depending on worker type
if worker_type == 'router':
worker_logname = "Router '{}'".format(worker_id)
elif worker_type == 'container':
worker_logname = "Container '{}'".format(worker_id)
elif worker_type == 'websocket-testee':
worker_logname = "WebSocketTestee '{}'".format(worker_id)
elif worker_type == 'guest':
worker_logname = "Guest '{}'".format(worker_id)
else:
raise Exception("logic error")
# any worker specific options
worker_options = worker.get('options', {})
# native worker processes: router, container, websocket-testee
if worker_type in ['router', 'container', 'websocket-testee']:
# start a new native worker process ..
if worker_type == 'router':
yield self._controller.start_router(worker_id, worker_options, details=call_details)
elif worker_type == 'container':
yield self._controller.start_container(worker_id, worker_options, details=call_details)
elif worker_type == 'websocket-testee':
yield self._controller.start_websocket_testee(worker_id, worker_options, details=call_details)
else:
raise Exception("logic error")
# setup native worker generic stuff
if 'pythonpath' in worker_options:
added_paths = yield self._controller.call('crossbar.worker.{}.add_pythonpath'.format(worker_id), worker_options['pythonpath'], options=call_options)
self.log.debug("{worker}: PYTHONPATH extended for {paths}",
worker=worker_logname, paths=added_paths)
if 'cpu_affinity' in worker_options:
new_affinity = yield self._controller.call('crossbar.worker.{}.set_cpu_affinity'.format(worker_id), worker_options['cpu_affinity'], options=call_options)
self.log.debug("{worker}: CPU affinity set to {affinity}",
worker=worker_logname, affinity=new_affinity)
if 'manhole' in worker:
yield self._controller.call('crossbar.worker.{}.start_manhole'.format(worker_id), worker['manhole'], options=call_options)
self.log.debug("{worker}: manhole started",
worker=worker_logname)
# setup router worker
if worker_type == 'router':
# start realms on router
for realm in worker.get('realms', []):
# start realm
if 'id' in realm:
realm_id = realm.pop('id')
else:
realm_id = 'realm-{:03d}'.format(self._realm_no)
self._realm_no += 1
yield self._controller.call('crossbar.worker.{}.start_router_realm'.format(worker_id), realm_id, realm, options=call_options)
self.log.info("{worker}: realm '{realm_id}' (named '{realm_name}') started",
worker=worker_logname, realm_id=realm_id, realm_name=realm['name'])
# add roles to realm
for role in realm.get('roles', []):
if 'id' in role:
role_id = role.pop('id')
else:
role_id = 'role-{:03d}'.format(self._role_no)
self._role_no += 1
yield self._controller.call('crossbar.worker.{}.start_router_realm_role'.format(worker_id), realm_id, role_id, role, options=call_options)
self.log.info(
"{logname}: role '{role}' (named '{role_name}') started on realm '{realm}'",
logname=worker_logname,
role=role_id,
role_name=role['name'],
realm=realm_id,
)
# start uplinks for realm
for uplink in realm.get('uplinks', []):
if 'id' in uplink:
uplink_id = uplink.pop('id')
else:
uplink_id = 'uplink-{:03d}'.format(self._uplink_no)
self._uplink_no += 1
yield self._controller.call('crossbar.worker.{}.start_router_realm_uplink'.format(worker_id), realm_id, uplink_id, uplink, options=call_options)
self.log.info(
"{logname}: uplink '{uplink}' started on realm '{realm}'",
logname=worker_logname,
uplink=uplink_id,
realm=realm_id,
)
# start connections (such as PostgreSQL database connection pools)
# to run embedded in the router
for connection in worker.get('connections', []):
if 'id' in connection:
connection_id = connection.pop('id')
else:
connection_id = 'connection-{:03d}'.format(self._connection_no)
self._connection_no += 1
yield self._controller.call('crossbar.worker.{}.start_connection'.format(worker_id), connection_id, connection, options=call_options)
self.log.info(
"{logname}: connection '{connection}' started",
logname=worker_logname,
connection=connection_id,
)
# start components to run embedded in the router
for component in worker.get('components', []):
if 'id' in component:
component_id = component.pop('id')
else:
component_id = 'component-{:03d}'.format(self._component_no)
self._component_no += 1
yield self._controller.call('crossbar.worker.{}.start_router_component'.format(worker_id), component_id, component, options=call_options)
self.log.info(
"{logname}: component '{component}' started",
logname=worker_logname,
component=component_id,
)
# start transports on router
for transport in worker['transports']:
if 'id' in transport:
transport_id = transport.pop('id')
else:
transport_id = 'transport-{:03d}'.format(self._transport_no)
self._transport_no += 1
yield self._controller.call('crossbar.worker.{}.start_router_transport'.format(worker_id), transport_id, transport, options=call_options)
self.log.info(
"{logname}: transport '{tid}' started",
logname=worker_logname,
tid=transport_id,
)
# setup container worker
elif worker_type == 'container':
# if components exit "very soon after" we try to
# start them, we consider that a failure and shut
# our node down. We remove this subscription 2
# seconds after we're done starting everything
# (see below). This is necessary as
# start_container_component returns as soon as
# we've established a connection to the component
def component_exited(info):
component_id = info.get("id")
self.log.critical("Component '{component_id}' failed to start; shutting down node.", component_id=component_id)
try:
self._reactor.stop()
except twisted.internet.error.ReactorNotRunning:
pass
topic = 'crossbar.worker.{}.container.on_component_stop'.format(worker_id)
component_stop_sub = yield self._controller.subscribe(component_exited, topic)
# start connections (such as PostgreSQL database connection pools)
# to run embedded in the container
#
for connection in worker.get('connections', []):
if 'id' in connection:
connection_id = connection.pop('id')
else:
connection_id = 'connection-{:03d}'.format(self._connection_no)
self._connection_no += 1
yield self._controller.call('crossbar.worker.{}.start_connection'.format(worker_id), connection_id, connection, options=call_options)
self.log.info(
"{logname}: connection '{connection}' started",
logname=worker_logname,
connection=connection_id,
)
# start components to run embedded in the container
#
for component in worker.get('components', []):
if 'id' in component:
component_id = component.pop('id')
else:
component_id = 'component-{:03d}'.format(self._component_no)
self._component_no += 1
yield self._controller.call('crossbar.worker.{}.start_container_component'.format(worker_id), component_id, component, options=call_options)
self.log.info("{worker}: component '{component_id}' started",
worker=worker_logname, component_id=component_id)
# after 2 seconds, consider all the application components running
self._reactor.callLater(2, component_stop_sub.unsubscribe)
# setup websocket-testee worker
elif worker_type == 'websocket-testee':
# start transport on websocket-testee
transport = worker['transport']
transport_id = 'transport-{:03d}'.format(self._transport_no)
self._transport_no = 1
yield self._controller.call('crossbar.worker.{}.start_websocket_testee_transport'.format(worker_id), transport_id, transport, options=call_options)
self.log.info(
"{logname}: transport '{tid}' started",
logname=worker_logname,
tid=transport_id,
)
else:
raise Exception("logic error")
elif worker_type == 'guest':
# start guest worker
#
yield self._controller.start_guest(worker_id, worker, details=call_details)
self.log.info("{worker}: started", worker=worker_logname)
else:
raise Exception("logic error")
self.log.info('Local node configuration applied.')
class Node:
"""
A Crossbar.io node is the running a controller process
and one or multiple worker processes.
A single Crossbar.io node runs exactly one instance of
this class, hence this class can be considered a system
singleton.
"""
def __init__(self, reactor, options):
"""
Ctor.
:param reactor: Reactor to run on.
:type reactor: obj
:param options: Options from command line.
:type options: obj
"""
self.debug = False
self.options = options
# the reactor under which we run
self._reactor = reactor
# shortname for reactor to run (when given via explicit option) or None
self._reactor_shortname = options.reactor
# node directory
self._cbdir = options.cbdir
# the node's name (must be unique within the management realm)
self._node_id = None
# the node's management realm
self._realm = None
# node controller session (a singleton ApplicationSession embedded
# in the node's management router)
self._controller = None
def start(self):
"""
Starts this node. This will start a node controller and then spawn new worker
processes as needed.
"""
# for now, a node is always started from a local configuration
#
configfile = os.path.join(self.options.cbdir, self.options.config)
log.msg("Starting from local configuration '{}'".format(configfile))
config = check_config_file(configfile, silence=True)
self.start_from_config(config)
def start_from_config(self, config):
controller_config = config.get('controller', {})
controller_options = controller_config.get('options', {})
controller_title = controller_options.get('title', 'crossbar-controller')
try:
import setproctitle
except ImportError:
log.msg("Warning, could not set process title (setproctitle not installed)")
else:
setproctitle.setproctitle(controller_title)
# the node's name (must be unique within the management realm)
if 'id' in controller_config:
self._node_id = controller_config['id']
else:
self._node_id = socket.gethostname()
# the node's management realm
self._realm = controller_config.get('realm', 'crossbar')
# the node controller singleton WAMP application session
#
# session_config = ComponentConfig(realm = options.realm, extra = options)
self._controller = NodeControllerSession(self)
# router and factory that creates router sessions
#
self._router_factory = RouterFactory(
options=RouterOptions(uri_check=RouterOptions.URI_CHECK_LOOSE),
debug=True)
self._router_session_factory = RouterSessionFactory(self._router_factory)
# add the node controller singleton session to the router
#
self._router_session_factory.add(self._controller)
# Detect WAMPlets
#
wamplets = self._controller._get_wamplets()
if len(wamplets) > 0:
log.msg("Detected {} WAMPlets in environment:".format(len(wamplets)))
for wpl in wamplets:
log.msg("WAMPlet {}.{}".format(wpl['dist'], wpl['name']))
else:
log.msg("No WAMPlets detected in enviroment.")
self.run_node_config(config)
def _start_from_local_config(self, configfile):
"""
Start Crossbar.io node from local configuration file.
"""
configfile = os.path.abspath(configfile)
log.msg("Starting from local config file '{}'".format(configfile))
try:
config = check_config_file(configfile, silence=True)
except Exception as e:
log.msg("Fatal: {}".format(e))
sys.exit(1)
else:
self.run_node_config(config)
@inlineCallbacks
def run_node_config(self, config):
try:
yield self._run_node_config(config)
except:
traceback.print_exc()
self._reactor.stop()
@inlineCallbacks
def _run_node_config(self, config):
"""
Setup node according to config provided.
"""
# fake call details information when calling into
# remoted procedure locally
#
call_details = CallDetails(caller=0)
controller = config.get('controller', {})
# start Manhole in node controller
#
if 'manhole' in controller:
yield self._controller.start_manhole(controller['manhole'], details=call_details)
# start local transport for management router
#
if 'transport' in controller:
yield self._controller.start_management_transport(controller['transport'], details=call_details)
# startup all workers
#
worker_no = 1
call_options = CallOptions(disclose_me=True)
for worker in config.get('workers', []):
# worker ID, type and logname
#
if 'id' in worker:
worker_id = worker.pop('id')
else:
worker_id = 'worker{}'.format(worker_no)
worker_no += 1
worker_type = worker['type']
worker_options = worker.get('options', {})
if worker_type == 'router':
worker_logname = "Router '{}'".format(worker_id)
elif worker_type == 'container':
worker_logname = "Container '{}'".format(worker_id)
elif worker_type == 'guest':
worker_logname = "Guest '{}'".format(worker_id)
else:
raise Exception("logic error")
# router/container
#
if worker_type in ['router', 'container']:
# start a new native worker process ..
#
if worker_type == 'router':
yield self._controller.start_router(worker_id, worker_options, details=call_details)
elif worker_type == 'container':
yield self._controller.start_container(worker_id, worker_options, details=call_details)
else:
raise Exception("logic error")
# setup native worker generic stuff
#
if 'pythonpath' in worker_options:
added_paths = yield self._controller.call('crossbar.node.{}.worker.{}.add_pythonpath'.format(self._node_id, worker_id), worker_options['pythonpath'], options=call_options)
if self.debug:
log.msg("{}: PYTHONPATH extended for {}".format(worker_logname, added_paths))
else:
log.msg("{}: PYTHONPATH extended".format(worker_logname))
if 'cpu_affinity' in worker_options:
new_affinity = yield self._controller.call('crossbar.node.{}.worker.{}.set_cpu_affinity'.format(self._node_id, worker_id), worker_options['cpu_affinity'], options=call_options)
log.msg("{}: CPU affinity set to {}".format(worker_logname, new_affinity))
if 'manhole' in worker:
yield self._controller.call('crossbar.node.{}.worker.{}.start_manhole'.format(self._node_id, worker_id), worker['manhole'], options=call_options)
log.msg("{}: manhole started".format(worker_logname))
# setup router worker
#
if worker_type == 'router':
# start realms on router
#
realm_no = 1
for realm in worker.get('realms', []):
if 'id' in realm:
realm_id = realm.pop('id')
else:
realm_id = 'realm{}'.format(realm_no)
realm_no += 1
# extract schema information from WAMP-flavored Markdown
#
schemas = None
if 'schemas' in realm:
schemas = {}
schema_pat = re.compile(r"```javascript(.*?)```", re.DOTALL)
cnt_files = 0
cnt_decls = 0
for schema_file in realm.pop('schemas'):
schema_file = os.path.join(self.options.cbdir, schema_file)
log.msg("{}: processing WAMP-flavored Markdown file {} for WAMP schema declarations".format(worker_logname, schema_file))
with open(schema_file, 'r') as f:
cnt_files += 1
for d in schema_pat.findall(f.read()):
try:
o = json.loads(d)
if isinstance(o, dict) and '$schema' in o and o['$schema'] == u'http://wamp.ws/schema#':
uri = o['uri']
if uri not in schemas:
schemas[uri] = {}
schemas[uri].update(o)
cnt_decls += 1
except Exception as e:
log.msg("{}: WARNING - failed to process declaration in {} - {}".format(worker_logname, schema_file, e))
log.msg("{}: processed {} files extracting {} schema declarations and {} URIs".format(worker_logname, cnt_files, cnt_decls, len(schemas)))
yield self._controller.call('crossbar.node.{}.worker.{}.start_router_realm'.format(self._node_id, worker_id), realm_id, realm, schemas, options=call_options)
log.msg("{}: realm '{}' (named '{}') started".format(worker_logname, realm_id, realm['name']))
# add roles to realm
#
role_no = 1
for role in realm.get('roles', []):
if 'id' in role:
role_id = role.pop('id')
else:
role_id = 'role{}'.format(role_no)
role_no += 1
yield self._controller.call('crossbar.node.{}.worker.{}.start_router_realm_role'.format(self._node_id, worker_id), realm_id, role_id, role, options=call_options)
log.msg("{}: role '{}' (named '{}') started on realm '{}'".format(worker_logname, role_id, role['name'], realm_id))
# start components to run embedded in the router
#
component_no = 1
for component in worker.get('components', []):
if 'id' in component:
component_id = component.pop('id')
else:
component_id = 'component{}'.format(component_no)
component_no += 1
yield self._controller.call('crossbar.node.{}.worker.{}.start_router_component'.format(self._node_id, worker_id), component_id, component, options=call_options)
log.msg("{}: component '{}' started".format(worker_logname, component_id))
# start transports on router
#
transport_no = 1
for transport in worker['transports']:
if 'id' in transport:
transport_id = transport.pop('id')
else:
transport_id = 'transport{}'.format(transport_no)
transport_no += 1
yield self._controller.call('crossbar.node.{}.worker.{}.start_router_transport'.format(self._node_id, worker_id), transport_id, transport, options=call_options)
log.msg("{}: transport '{}' started".format(worker_logname, transport_id))
# setup container worker
#
elif worker_type == 'container':
component_no = 1
# if components exit "very soon after" we try to
# start them, we consider that a failure and shut
# our node down. We remove this subscription 2
# seconds after we're done starting everything
# (see below). This is necessary as
# start_container_component returns as soon as
# we've established a connection to the component
def component_exited(info):
dead_comp = info['id']
log.msg("Component '{}' failed to start; shutting down node.".format(dead_comp))
if self._reactor.running:
self._reactor.stop()
topic = 'crossbar.node.{}.worker.{}.container.on_component_stop'.format(self._node_id, worker_id)
component_stop_sub = yield self._controller.subscribe(component_exited, topic)
for component in worker.get('components', []):
if 'id' in component:
component_id = component.pop('id')
else:
component_id = 'component{}'.format(component_no)
component_no += 1
yield self._controller.call('crossbar.node.{}.worker.{}.start_container_component'.format(self._node_id, worker_id), component_id, component, options=call_options)
log.msg("{}: component '{}' started".format(worker_logname, component_id))
# after 2 seconds, consider all the application components running
self._reactor.callLater(2, component_stop_sub.unsubscribe)
else:
raise Exception("logic error")
elif worker_type == 'guest':
# start guest worker
#
yield self._controller.start_guest(worker_id, worker, details=call_details)
log.msg("{}: started".format(worker_logname))
else:
raise Exception("logic error")
class Node:
"""
A Crossbar.io node is the running a controller process
and one or multiple worker processes.
A single Crossbar.io node runs exactly one instance of
this class, hence this class can be considered a system
singleton.
"""
def __init__(self, reactor, options):
"""
Ctor.
:param reactor: Reactor to run on.
:type reactor: obj
:param options: Options from command line.
:type options: obj
"""
self.debug = False
self.options = options
## the reactor under which we run
self._reactor = reactor
## shortname for reactor to run (when given via explicit option) or None
self._reactor_shortname = options.reactor
## node directory
self._cbdir = options.cbdir
## the node's name (must be unique within the management realm)
self._node_id = None
## the node's management realm
self._realm = None
## node controller session (a singleton ApplicationSession embedded
## in the node's management router)
self._controller = None
def start(self):
"""
Starts this node. This will start a node controller and then spawn new worker
processes as needed.
"""
## for now, a node is always started from a local configuration
##
configfile = os.path.join(self.options.cbdir, self.options.config)
log.msg("Starting from local configuration '{}'".format(configfile))
config = checkconfig.check_config_file(configfile, silence = True)
self.start_from_config(config)
def start_from_config(self, config):
controller_config = config.get('controller', {})
controller_options = controller_config.get('options', {})
controller_title = controller_options.get('title', 'crossbar-controller')
try:
import setproctitle
except ImportError:
log.msg("Warning, could not set process title (setproctitle not installed)")
else:
setproctitle.setproctitle(controller_title)
## the node's name (must be unique within the management realm)
if 'id' in controller_config:
self._node_id = controller_config['id']
else:
self._node_id = socket.gethostname()
## the node's management realm
self._realm = controller_config.get('realm', 'crossbar')
## the node controller singleton WAMP application session
##
#session_config = ComponentConfig(realm = options.realm, extra = options)
self._controller = NodeControllerSession(self)
## router and factory that creates router sessions
##
self._router_factory = RouterFactory(
options = RouterOptions(uri_check = RouterOptions.URI_CHECK_LOOSE),
debug = False)
self._router_session_factory = RouterSessionFactory(self._router_factory)
## add the node controller singleton session to the router
##
self._router_session_factory.add(self._controller)
## Detect WAMPlets
##
wamplets = self._controller._get_wamplets()
if len(wamplets) > 0:
log.msg("Detected {} WAMPlets in environment:".format(len(wamplets)))
for wpl in wamplets:
log.msg("WAMPlet {}.{}".format(wpl['dist'], wpl['name']))
else:
log.msg("No WAMPlets detected in enviroment.")
self.run_node_config(config)
def _start_from_local_config(self, configfile):
"""
Start Crossbar.io node from local configuration file.
"""
configfile = os.path.abspath(configfile)
log.msg("Starting from local config file '{}'".format(configfile))
try:
config = controller.config.check_config_file(configfile, silence = True)
except Exception as e:
log.msg("Fatal: {}".format(e))
sys.exit(1)
else:
self.run_node_config(config)
@inlineCallbacks
def run_node_config(self, config):
try:
yield self._run_node_config(config)
except:
traceback.print_exc()
self._reactor.stop()
@inlineCallbacks
def _run_node_config(self, config):
"""
Setup node according to config provided.
"""
## fake call details information when calling into
## remoted procedure locally
##
call_details = CallDetails(caller = 0, authid = 'node')
controller = config.get('controller', {})
## start Manhole in node controller
##
if 'manhole' in controller:
yield self._controller.start_manhole(controller['manhole'], details = call_details)
## start local transport for management router
##
if 'transport' in controller:
yield self._controller.start_management_transport(controller['transport'], details = call_details)
## startup all workers
##
worker_no = 1
for worker in config.get('workers', []):
## worker ID, type and logname
##
if 'id' in worker:
worker_id = worker.pop('id')
else:
worker_id = 'worker{}'.format(worker_no)
worker_no += 1
worker_type = worker['type']
worker_options = worker.get('options', {})
if worker_type == 'router':
worker_logname = "Router '{}'".format(worker_id)
elif worker_type == 'container':
worker_logname = "Container '{}'".format(worker_id)
elif worker_type == 'guest':
worker_logname = "Guest '{}'".format(worker_id)
else:
raise Exception("logic error")
## router/container
##
if worker_type in ['router', 'container']:
## start a new native worker process ..
##
if worker_type == 'router':
yield self._controller.start_router(worker_id, worker_options, details = call_details)
elif worker_type == 'container':
yield self._controller.start_container(worker_id, worker_options, details = call_details)
else:
raise Exception("logic error")
## setup native worker generic stuff
##
if 'pythonpath' in worker_options:
added_paths = yield self._controller.call('crossbar.node.{}.worker.{}.add_pythonpath'.format(self._node_id, worker_id), worker_options['pythonpath'])
if self.debug:
log.msg("{}: PYTHONPATH extended for {}".format(worker_logname, added_paths))
else:
log.msg("{}: PYTHONPATH extended".format(worker_logname))
if 'cpu_affinity' in worker_options:
new_affinity = yield self._controller.call('crossbar.node.{}.worker.{}.set_cpu_affinity'.format(self._node_id, worker_id), worker_options['cpu_affinity'])
log.msg("{}: CPU affinity set to {}".format(worker_logname, new_affinity))
if 'manhole' in worker:
yield self._controller.call('crossbar.node.{}.worker.{}.start_manhole'.format(self._node_id, worker_id), worker['manhole'])
log.msg("{}: manhole started".format(worker_logname))
## setup router worker
##
if worker_type == 'router':
## start realms on router
##
realm_no = 1
for realm in worker.get('realms', []):
if 'id' in realm:
realm_id = realm.pop('id')
else:
realm_id = 'realm{}'.format(realm_no)
realm_no += 1
## extract schema information from WAMP-flavored Markdown
##
schemas = None
if 'schemas' in realm:
schemas = {}
schema_pat = re.compile(r"```javascript(.*?)```", re.DOTALL)
cnt_files = 0
cnt_decls = 0
for schema_file in realm.pop('schemas'):
schema_file = os.path.join(self.options.cbdir, schema_file)
log.msg("{}: processing WAMP-flavored Markdown file {} for WAMP schema declarations".format(worker_logname, schema_file))
with open(schema_file, 'r') as f:
cnt_files += 1
for d in schema_pat.findall(f.read()):
try:
o = json.loads(d)
if type(o) == dict and '$schema' in o and o['$schema'] == u'http://wamp.ws/schema#':
uri = o['uri']
if not uri in schemas:
schemas[uri] = {}
schemas[uri].update(o)
cnt_decls += 1
except Exception as e:
log.msg("{}: WARNING - failed to process declaration in {} - {}".format(worker_logname, schema_file, e))
log.msg("{}: processed {} files extracting {} schema declarations and {} URIs".format(worker_logname, cnt_files, cnt_decls, len(schemas)))
yield self._controller.call('crossbar.node.{}.worker.{}.start_router_realm'.format(self._node_id, worker_id), realm_id, realm, schemas)
log.msg("{}: realm '{}' started".format(worker_logname, realm_id))
## add roles to realm
##
role_no = 1
for role in realm.get('roles', []):
if 'id' in role:
role_id = role.pop('id')
else:
role_id = 'role{}'.format(role_no)
role_no += 1
yield self._controller.call('crossbar.node.{}.worker.{}.start_router_realm_role'.format(self._node_id, worker_id), realm_id, role_id, role)
log.msg("{}: role '{}' started on realm '{}'".format(worker_logname, role_id, realm_id))
## start components to run embedded in the router
##
component_no = 1
for component in worker.get('components', []):
if 'id' in component:
component_id = component.pop('id')
else:
component_id = 'component{}'.format(component_no)
component_no += 1
yield self._controller.call('crossbar.node.{}.worker.{}.start_router_component'.format(self._node_id, worker_id), component_id, component)
log.msg("{}: component '{}' started".format(worker_logname, component_id))
## start transports on router
##
transport_no = 1
for transport in worker['transports']:
if 'id' in transport:
transport_id = transport.pop('id')
else:
transport_id = 'transport{}'.format(transport_no)
transport_no += 1
yield self._controller.call('crossbar.node.{}.worker.{}.start_router_transport'.format(self._node_id, worker_id), transport_id, transport)
log.msg("{}: transport '{}' started".format(worker_logname, transport_id))
## setup container worker
##
elif worker_type == 'container':
component_no = 1
for component in worker.get('components', []):
if 'id' in component:
component_id = component.pop('id')
else:
component_id = 'component{}'.format(component_no)
component_no += 1
yield self._controller.call('crossbar.node.{}.worker.{}.start_container_component'.format(self._node_id, worker_id), component_id, component)
log.msg("{}: component '{}' started".format(worker_logname, component_id))
else:
raise Exception("logic error")
elif worker_type == 'guest':
## start guest worker
##
yield self._controller.start_guest(worker_id, worker, details = call_details)
log.msg("{}: started".format(worker_logname))
else:
raise Exception("logic error")
class Node(object):
"""
A Crossbar.io node is the running a controller process and one or multiple
worker processes.
A single Crossbar.io node runs exactly one instance of this class, hence
this class can be considered a system singleton.
"""
log = make_logger()
def __init__(self, cbdir=None, reactor=None):
"""
:param cbdir: The node directory to run from.
:type cbdir: unicode
:param reactor: Reactor to run on.
:type reactor: obj or None
"""
# node directory
self._cbdir = cbdir or u'.'
# reactor we should run on
if reactor is None:
from twisted.internet import reactor
self._reactor = reactor
# the node's management realm when running in managed mode (this comes from CDC!)
self._management_realm = None
# the node's ID when running in managed mode (this comes from CDC!)
self._node_id = None
# node extra when running in managed mode (this comes from CDC!)
self._node_extra = None
# the node controller realm
self._realm = u'crossbar'
# config of this node.
self._config = None
# node private key autobahn.wamp.cryptosign.SigningKey
self._node_key = None
# node controller session (a singleton ApplicationSession embedded
# in the local node router)
self._controller = None
# when running in managed mode, this will hold the bridge session
# attached to the local management router
self._bridge_session = None
# when running in managed mode, this will hold the uplink session to CDC
self._manager = None
# node shutdown triggers, one or more of checkconfig.NODE_SHUTDOWN_MODES
self._node_shutdown_triggers = [
checkconfig.NODE_SHUTDOWN_ON_WORKER_EXIT
]
# map from router worker IDs to
self._realm_templates = {}
# for node elements started under specific IDs, and where
# the node configuration does not specify an ID, use a generic
# name numbered sequentially using the counters here
self._worker_no = 1
self._realm_no = 1
self._role_no = 1
self._connection_no = 1
self._transport_no = 1
self._component_no = 1
def maybe_generate_key(self,
cbdir,
privkey_path=u'key.priv',
pubkey_path=u'key.pub'):
privkey_path = os.path.join(cbdir, privkey_path)
pubkey_path = os.path.join(cbdir, pubkey_path)
if os.path.exists(privkey_path):
# node private key seems to exist already .. check!
priv_tags = _parse_keyfile(privkey_path, private=True)
for tag in [
u'creator', u'created-at', u'machine-id',
u'public-key-ed25519', u'private-key-ed25519'
]:
if tag not in priv_tags:
raise Exception(
"Corrupt node private key file {} - {} tag not found".
format(privkey_path, tag))
privkey_hex = priv_tags[u'private-key-ed25519']
privkey = SigningKey(privkey_hex, encoder=HexEncoder)
pubkey = privkey.verify_key
pubkey_hex = pubkey.encode(encoder=HexEncoder).decode('ascii')
if priv_tags[u'public-key-ed25519'] != pubkey_hex:
raise Exception((
"Inconsistent node private key file {} - public-key-ed25519 doesn't"
" correspond to private-key-ed25519").format(pubkey_path))
if os.path.exists(pubkey_path):
pub_tags = _parse_keyfile(pubkey_path, private=False)
for tag in [
u'creator', u'created-at', u'machine-id',
u'public-key-ed25519'
]:
if tag not in pub_tags:
raise Exception(
"Corrupt node public key file {} - {} tag not found"
.format(pubkey_path, tag))
if pub_tags[u'public-key-ed25519'] != pubkey_hex:
raise Exception((
"Inconsistent node public key file {} - public-key-ed25519 doesn't"
" correspond to private-key-ed25519"
).format(pubkey_path))
else:
self.log.info(
"Node public key file {pub_path} not found - re-creating from node private key file {priv_path}",
pub_path=pubkey_path,
priv_path=privkey_path,
)
pub_tags = OrderedDict([
(u'creator', priv_tags[u'creator']),
(u'created-at', priv_tags[u'created-at']),
(u'machine-id', priv_tags[u'machine-id']),
(u'public-key-ed25519', pubkey_hex),
])
msg = u'Crossbar.io node public key\n\n'
_write_node_key(pubkey_path, pub_tags, msg)
self.log.debug("Node key already exists (public key: {hex})",
hex=pubkey_hex)
else:
# node private key does not yet exist: generate one
privkey = SigningKey.generate()
privkey_hex = privkey.encode(encoder=HexEncoder).decode('ascii')
pubkey = privkey.verify_key
pubkey_hex = pubkey.encode(encoder=HexEncoder).decode('ascii')
# first, write the public file
tags = OrderedDict([
(u'creator', _creator()),
(u'created-at', utcnow()),
(u'machine-id', _machine_id()),
(u'public-key-ed25519', pubkey_hex),
])
msg = u'Crossbar.io node public key\n\n'
_write_node_key(pubkey_path, tags, msg)
# now, add the private key and write the private file
tags[u'private-key-ed25519'] = privkey_hex
msg = u'Crossbar.io node private key - KEEP THIS SAFE!\n\n'
_write_node_key(privkey_path, tags, msg)
self.log.info("New node key pair generated!")
# fix file permissions on node public/private key files
# note: we use decimals instead of octals as octal literals have changed between Py2/3
#
if os.stat(pubkey_path
).st_mode & 511 != 420: # 420 (decimal) == 0644 (octal)
os.chmod(pubkey_path, 420)
self.log.info("File permissions on node public key fixed!")
if os.stat(privkey_path
).st_mode & 511 != 384: # 384 (decimal) == 0600 (octal)
os.chmod(privkey_path, 384)
self.log.info("File permissions on node private key fixed!")
self._node_key = cryptosign.SigningKey(privkey)
return pubkey_hex
def load(self, configfile=None):
"""
Check and load the node configuration (usually, from ".crossbar/config.json")
or load built-in CDC default config.
"""
if configfile:
configpath = os.path.join(self._cbdir, configfile)
self.log.debug("Loading node configuration from '{configpath}' ..",
configpath=configpath)
# the following will read the config, check the config and replace
# environment variable references in configuration values ("${MYVAR}") and
# finally return the parsed configuration object
self._config = checkconfig.check_config_file(configpath)
self.log.info("Node configuration loaded from '{configfile}'",
configfile=configfile)
else:
self._config = {u'version': 2, u'controller': {}, u'workers': []}
checkconfig.check_config(self._config)
self.log.info("Node configuration loaded from built-in config.")
@inlineCallbacks
def start(self, cdc_mode=False):
"""
Starts this node. This will start a node controller and then spawn new worker
processes as needed.
"""
if not self._config:
raise Exception("No node configuration loaded")
if not cdc_mode and not self._config.get(
"controller", {}) and not self._config.get("workers", {}):
self.log.warn(
("You seem to have no controller config or workers, nor are "
"starting up in CDC mode. Check your config exists, or pass "
"--cdc to `crossbar start`."))
try:
self._reactor.stop()
except twisted.internet.error.ReactorNotRunning:
pass
return
# get controller config/options
#
controller_config = self._config.get('controller', {})
controller_options = controller_config.get('options', {})
# set controller process title
#
try:
import setproctitle
except ImportError:
self.log.warn(
"Warning, could not set process title (setproctitle not installed)"
)
else:
setproctitle.setproctitle(
controller_options.get('title', 'crossbar-controller'))
# router and factory that creates router sessions
#
self._router_factory = RouterFactory()
self._router_session_factory = RouterSessionFactory(
self._router_factory)
# create a new router for the realm
#
rlm_config = {'name': self._realm}
rlm = RouterRealm(None, rlm_config)
router = self._router_factory.start_realm(rlm)
# always add a realm service session
#
cfg = ComponentConfig(self._realm)
rlm.session = RouterServiceSession(cfg, router)
self._router_session_factory.add(rlm.session, authrole=u'trusted')
# add a router bridge session when running in managed mode
#
if cdc_mode:
self._bridge_session = NodeManagementBridgeSession(cfg)
self._router_session_factory.add(self._bridge_session,
authrole=u'trusted')
else:
self._bridge_session = None
# Node shutdown mode
#
if cdc_mode:
# in managed mode, a node - by default - only shuts down when explicitly asked to,
# or upon a fatal error in the node controller
self._node_shutdown_triggers = [
checkconfig.NODE_SHUTDOWN_ON_SHUTDOWN_REQUESTED
]
else:
# in standalone mode, a node - by default - is immediately shutting down whenever
# a worker exits (successfully or with error)
self._node_shutdown_triggers = [
checkconfig.NODE_SHUTDOWN_ON_WORKER_EXIT
]
# allow to override node shutdown triggers
#
if 'shutdown' in controller_options:
self.log.info(
"Overriding default node shutdown triggers with {triggers} from node config",
triggers=controller_options['shutdown'])
self._node_shutdown_triggers = controller_options['shutdown']
else:
self.log.info("Using default node shutdown triggers {triggers}",
triggers=self._node_shutdown_triggers)
# add the node controller singleton session
#
self._controller = NodeControllerSession(self)
self._router_session_factory.add(self._controller, authrole=u'trusted')
# detect WAMPlets (FIXME: remove this!)
#
wamplets = self._controller._get_wamplets()
if len(wamplets) > 0:
self.log.info("Detected {wamplets} WAMPlets in environment:",
wamplets=len(wamplets))
for wpl in wamplets:
self.log.info("WAMPlet {dist}.{name}",
dist=wpl['dist'],
name=wpl['name'])
else:
self.log.debug("No WAMPlets detected in enviroment.")
panic = False
try:
# startup the node from local node configuration
#
yield self._startup(self._config)
# connect to CDC when running in managed mode
#
if cdc_mode:
cdc_config = controller_config.get(
'cdc',
{
# CDC connecting transport
u'transport': {
u'type': u'websocket',
u'url': u'wss://cdc.crossbario.com/ws',
u'endpoint': {
u'type': u'tcp',
u'host': u'cdc.crossbario.com',
u'port': 443,
u'timeout': 5,
u'tls': {
u'hostname': u'cdc.crossbario.com'
}
}
}
})
transport = cdc_config[u'transport']
hostname = None
if u'tls' in transport[u'endpoint']:
transport[u'endpoint'][u'tls'][u'hostname']
runner = ApplicationRunner(
url=transport['url'],
realm=None,
extra=None,
ssl=optionsForClientTLS(hostname) if hostname else None,
)
def make(config):
# extra info forwarded to CDC client session
extra = {
'node': self,
'on_ready': Deferred(),
'on_exit': Deferred(),
'node_key': self._node_key,
}
@inlineCallbacks
def on_ready(res):
self._manager, self._management_realm, self._node_id, self._node_extra = res
if self._bridge_session:
try:
yield self._bridge_session.attach_manager(
self._manager, self._management_realm,
self._node_id)
status = yield self._manager.call(
u'cdc.remote.status@1')
except:
self.log.failure()
else:
self.log.info(
'Connected to CDC for management realm "{realm}" (current time is {now})',
realm=self._management_realm,
now=status[u'now'])
else:
self.log.warn(
'Uplink CDC session established, but no bridge session setup!'
)
@inlineCallbacks
def on_exit(res):
if self._bridge_session:
try:
yield self._bridge_session.detach_manager()
except:
self.log.failure()
else:
self.log.info(
'Disconnected from CDC for management realm "{realm}"',
realm=self._management_realm)
else:
self.log.warn(
'Uplink CDC session lost, but no bridge session setup!'
)
self._manager, self._management_realm, self._node_id, self._node_extra = None, None, None, None
extra['on_ready'].addCallback(on_ready)
extra['on_exit'].addCallback(on_exit)
config = ComponentConfig(extra=extra)
session = NodeManagementSession(config)
return session
self.log.info("Connecting to CDC at '{url}' ..",
url=transport[u'url'])
yield runner.run(make,
start_reactor=False,
auto_reconnect=True)
# Notify systemd that crossbar is fully up and running
# (this has no effect on non-systemd platforms)
try:
import sdnotify
sdnotify.SystemdNotifier().notify("READY=1")
except:
pass
except ApplicationError as e:
panic = True
self.log.error("{msg}", msg=e.error_message())
except Exception:
panic = True
traceback.print_exc()
if panic:
try:
self._reactor.stop()
except twisted.internet.error.ReactorNotRunning:
pass
@inlineCallbacks
def _startup(self, config):
"""
Startup elements in the node as specified in the provided node configuration.
"""
self.log.info('Configuring node from local configuration ...')
# call options we use to call into the local node management API
call_options = CallOptions()
# fake call details we use to call into the local node management API
call_details = CallDetails(caller=0)
# get contoller configuration subpart
controller = config.get('controller', {})
# start Manhole in node controller
if 'manhole' in controller:
yield self._controller.start_manhole(controller['manhole'],
details=call_details)
# startup all workers
workers = config.get('workers', [])
if len(workers):
self.log.info('Starting {nworkers} workers ...',
nworkers=len(workers))
else:
self.log.info('No workers configured!')
for worker in workers:
# worker ID
if 'id' in worker:
worker_id = worker.pop('id')
else:
worker_id = 'worker-{:03d}'.format(self._worker_no)
self._worker_no += 1
# worker type - a type of working process from the following fixed list
worker_type = worker['type']
assert (worker_type
in ['router', 'container', 'guest', 'websocket-testee'])
# set logname depending on worker type
if worker_type == 'router':
worker_logname = "Router '{}'".format(worker_id)
elif worker_type == 'container':
worker_logname = "Container '{}'".format(worker_id)
elif worker_type == 'websocket-testee':
worker_logname = "WebSocketTestee '{}'".format(worker_id)
elif worker_type == 'guest':
worker_logname = "Guest '{}'".format(worker_id)
else:
raise Exception("logic error")
# any worker specific options
worker_options = worker.get('options', {})
# native worker processes: router, container, websocket-testee
if worker_type in ['router', 'container', 'websocket-testee']:
# start a new native worker process ..
if worker_type == 'router':
yield self._controller.start_router(worker_id,
worker_options,
details=call_details)
elif worker_type == 'container':
yield self._controller.start_container(
worker_id, worker_options, details=call_details)
elif worker_type == 'websocket-testee':
yield self._controller.start_websocket_testee(
worker_id, worker_options, details=call_details)
else:
raise Exception("logic error")
# setup native worker generic stuff
if 'pythonpath' in worker_options:
added_paths = yield self._controller.call(
'crossbar.worker.{}.add_pythonpath'.format(worker_id),
worker_options['pythonpath'],
options=call_options)
self.log.debug("{worker}: PYTHONPATH extended for {paths}",
worker=worker_logname,
paths=added_paths)
if 'cpu_affinity' in worker_options:
new_affinity = yield self._controller.call(
'crossbar.worker.{}.set_cpu_affinity'.format(
worker_id),
worker_options['cpu_affinity'],
options=call_options)
self.log.debug("{worker}: CPU affinity set to {affinity}",
worker=worker_logname,
affinity=new_affinity)
if 'manhole' in worker:
yield self._controller.call(
'crossbar.worker.{}.start_manhole'.format(worker_id),
worker['manhole'],
options=call_options)
self.log.debug("{worker}: manhole started",
worker=worker_logname)
# setup router worker
if worker_type == 'router':
# start realms on router
for realm in worker.get('realms', []):
# start realm
if 'id' in realm:
realm_id = realm.pop('id')
else:
realm_id = 'realm-{:03d}'.format(self._realm_no)
self._realm_no += 1
yield self._controller.call(
'crossbar.worker.{}.start_router_realm'.format(
worker_id),
realm_id,
realm,
options=call_options)
self.log.info(
"{worker}: realm '{realm_id}' (named '{realm_name}') started",
worker=worker_logname,
realm_id=realm_id,
realm_name=realm['name'])
# add roles to realm
for role in realm.get('roles', []):
if 'id' in role:
role_id = role.pop('id')
else:
role_id = 'role-{:03d}'.format(self._role_no)
self._role_no += 1
yield self._controller.call(
'crossbar.worker.{}.start_router_realm_role'.
format(worker_id),
realm_id,
role_id,
role,
options=call_options)
self.log.info(
"{logname}: role '{role}' (named '{role_name}') started on realm '{realm}'",
logname=worker_logname,
role=role_id,
role_name=role['name'],
realm=realm_id,
)
# start uplinks for realm
for uplink in realm.get('uplinks', []):
if 'id' in uplink:
uplink_id = uplink.pop('id')
else:
uplink_id = 'uplink-{:03d}'.format(
self._uplink_no)
self._uplink_no += 1
yield self._controller.call(
'crossbar.worker.{}.start_router_realm_uplink'.
format(worker_id),
realm_id,
uplink_id,
uplink,
options=call_options)
self.log.info(
"{logname}: uplink '{uplink}' started on realm '{realm}'",
logname=worker_logname,
uplink=uplink_id,
realm=realm_id,
)
# start connections (such as PostgreSQL database connection pools)
# to run embedded in the router
for connection in worker.get('connections', []):
if 'id' in connection:
connection_id = connection.pop('id')
else:
connection_id = 'connection-{:03d}'.format(
self._connection_no)
self._connection_no += 1
yield self._controller.call(
'crossbar.worker.{}.start_connection'.format(
worker_id),
connection_id,
connection,
options=call_options)
self.log.info(
"{logname}: connection '{connection}' started",
logname=worker_logname,
connection=connection_id,
)
# start components to run embedded in the router
for component in worker.get('components', []):
if 'id' in component:
component_id = component.pop('id')
else:
component_id = 'component-{:03d}'.format(
self._component_no)
self._component_no += 1
yield self._controller.call(
'crossbar.worker.{}.start_router_component'.format(
worker_id),
component_id,
component,
options=call_options)
self.log.info(
"{logname}: component '{component}' started",
logname=worker_logname,
component=component_id,
)
# start transports on router
for transport in worker['transports']:
if 'id' in transport:
transport_id = transport.pop('id')
else:
transport_id = 'transport-{:03d}'.format(
self._transport_no)
self._transport_no += 1
yield self._controller.call(
'crossbar.worker.{}.start_router_transport'.format(
worker_id),
transport_id,
transport,
options=call_options)
self.log.info(
"{logname}: transport '{tid}' started",
logname=worker_logname,
tid=transport_id,
)
# setup container worker
elif worker_type == 'container':
# if components exit "very soon after" we try to
# start them, we consider that a failure and shut
# our node down. We remove this subscription 2
# seconds after we're done starting everything
# (see below). This is necessary as
# start_container_component returns as soon as
# we've established a connection to the component
def component_exited(info):
component_id = info.get("id")
self.log.critical(
"Component '{component_id}' failed to start; shutting down node.",
component_id=component_id)
try:
self._reactor.stop()
except twisted.internet.error.ReactorNotRunning:
pass
topic = 'crossbar.worker.{}.container.on_component_stop'.format(
worker_id)
component_stop_sub = yield self._controller.subscribe(
component_exited, topic)
# start connections (such as PostgreSQL database connection pools)
# to run embedded in the container
#
for connection in worker.get('connections', []):
if 'id' in connection:
connection_id = connection.pop('id')
else:
connection_id = 'connection-{:03d}'.format(
self._connection_no)
self._connection_no += 1
yield self._controller.call(
'crossbar.worker.{}.start_connection'.format(
worker_id),
connection_id,
connection,
options=call_options)
self.log.info(
"{logname}: connection '{connection}' started",
logname=worker_logname,
connection=connection_id,
)
# start components to run embedded in the container
#
for component in worker.get('components', []):
if 'id' in component:
component_id = component.pop('id')
else:
component_id = 'component-{:03d}'.format(
self._component_no)
self._component_no += 1
yield self._controller.call(
'crossbar.worker.{}.start_container_component'.
format(worker_id),
component_id,
component,
options=call_options)
self.log.info(
"{worker}: component '{component_id}' started",
worker=worker_logname,
component_id=component_id)
# after 2 seconds, consider all the application components running
self._reactor.callLater(2, component_stop_sub.unsubscribe)
# setup websocket-testee worker
elif worker_type == 'websocket-testee':
# start transport on websocket-testee
transport = worker['transport']
transport_id = 'transport-{:03d}'.format(
self._transport_no)
self._transport_no = 1
yield self._controller.call(
'crossbar.worker.{}.start_websocket_testee_transport'.
format(worker_id),
transport_id,
transport,
options=call_options)
self.log.info(
"{logname}: transport '{tid}' started",
logname=worker_logname,
tid=transport_id,
)
else:
raise Exception("logic error")
elif worker_type == 'guest':
# start guest worker
#
yield self._controller.start_guest(worker_id,
worker,
details=call_details)
self.log.info("{worker}: started", worker=worker_logname)
else:
raise Exception("logic error")
self.log.info('Local node configuration applied.')
def start(self):
"""
Starts this node. This will start a node controller and then spawn new worker
processes as needed.
"""
if not self._config:
self.check_config()
controller_config = self._config.get('controller', {})
controller_options = controller_config.get('options', {})
controller_title = controller_options.get('title',
'crossbar-controller')
try:
import setproctitle
except ImportError:
self.log.warn(
"Warning, could not set process title (setproctitle not installed)"
)
else:
setproctitle.setproctitle(controller_title)
# the node's name (must be unique within the management realm)
if 'id' in controller_config:
self._node_id = controller_config['id']
else:
self._node_id = socket.gethostname()
if 'manager' in controller_config:
extra = {
'onready': Deferred(),
# authentication information for connecting to uplinkg CDC router
# using WAMP-CRA authentication
#
'authid': self._node_id,
'authkey': controller_config['manager']['key']
}
realm = controller_config['manager']['realm']
transport = controller_config['manager']['transport']
runner = ApplicationRunner(url=transport['url'],
realm=realm,
extra=extra,
debug_wamp=False)
runner.run(NodeManagementSession, start_reactor=False)
# wait until we have attached to the uplink CDC
self._management_session = yield extra['onready']
self.log.info(
"Node is connected to Crossbar.io DevOps Center (CDC)")
else:
self._management_session = None
# the node's management realm
self._realm = controller_config.get('realm', 'crossbar')
# router and factory that creates router sessions
#
self._router_factory = RouterFactory()
self._router_session_factory = RouterSessionFactory(
self._router_factory)
rlm = RouterRealm(None, {'name': self._realm})
# create a new router for the realm
router = self._router_factory.start_realm(rlm)
# add a router/realm service session
cfg = ComponentConfig(self._realm)
rlm.session = RouterServiceSession(cfg, router)
self._router_session_factory.add(rlm.session, authrole=u'trusted')
if self._management_session:
self._bridge_session = NodeManagementBridgeSession(
cfg, self._management_session)
self._router_session_factory.add(self._bridge_session,
authrole=u'trusted')
else:
self._bridge_session = None
# the node controller singleton WAMP application session
#
self._controller = NodeControllerSession(self)
# add the node controller singleton session to the router
#
self._router_session_factory.add(self._controller, authrole=u'trusted')
# Detect WAMPlets
#
wamplets = self._controller._get_wamplets()
if len(wamplets) > 0:
self.log.info("Detected {wamplets} WAMPlets in environment:",
wamplets=len(wamplets))
for wpl in wamplets:
self.log.info("WAMPlet {dist}.{name}",
dist=wpl['dist'],
name=wpl['name'])
else:
self.log.info("No WAMPlets detected in enviroment.")
panic = False
try:
yield self._startup(self._config)
except ApplicationError as e:
panic = True
for line in e.args[0].strip().splitlines():
self.log.error(line)
except Exception:
panic = True
traceback.print_exc()
if panic:
try:
self._reactor.stop()
except twisted.internet.error.ReactorNotRunning:
pass
def start(self, cdc_mode=False):
"""
Starts this node. This will start a node controller and then spawn new worker
processes as needed.
"""
if not self._config:
raise Exception("No node configuration loaded")
if not cdc_mode and not self._config.get(
"controller", {}) and not self._config.get("workers", {}):
self.log.warn(
("You seem to have no controller config or workers, nor are "
"starting up in CDC mode. Check your config exists, or pass "
"--cdc to `crossbar start`."))
try:
self._reactor.stop()
except twisted.internet.error.ReactorNotRunning:
pass
return
# get controller config/options
#
controller_config = self._config.get('controller', {})
controller_options = controller_config.get('options', {})
# set controller process title
#
try:
import setproctitle
except ImportError:
self.log.warn(
"Warning, could not set process title (setproctitle not installed)"
)
else:
setproctitle.setproctitle(
controller_options.get('title', 'crossbar-controller'))
# router and factory that creates router sessions
#
self._router_factory = RouterFactory()
self._router_session_factory = RouterSessionFactory(
self._router_factory)
# create a new router for the realm
#
rlm_config = {'name': self._realm}
rlm = RouterRealm(None, rlm_config)
router = self._router_factory.start_realm(rlm)
# always add a realm service session
#
cfg = ComponentConfig(self._realm)
rlm.session = RouterServiceSession(cfg, router)
self._router_session_factory.add(rlm.session, authrole=u'trusted')
# add a router bridge session when running in managed mode
#
if cdc_mode:
self._bridge_session = NodeManagementBridgeSession(cfg)
self._router_session_factory.add(self._bridge_session,
authrole=u'trusted')
else:
self._bridge_session = None
# Node shutdown mode
#
if cdc_mode:
# in managed mode, a node - by default - only shuts down when explicitly asked to,
# or upon a fatal error in the node controller
self._node_shutdown_triggers = [
checkconfig.NODE_SHUTDOWN_ON_SHUTDOWN_REQUESTED
]
else:
# in standalone mode, a node - by default - is immediately shutting down whenever
# a worker exits (successfully or with error)
self._node_shutdown_triggers = [
checkconfig.NODE_SHUTDOWN_ON_WORKER_EXIT
]
# allow to override node shutdown triggers
#
if 'shutdown' in controller_options:
self.log.info(
"Overriding default node shutdown triggers with {triggers} from node config",
triggers=controller_options['shutdown'])
self._node_shutdown_triggers = controller_options['shutdown']
else:
self.log.info("Using default node shutdown triggers {triggers}",
triggers=self._node_shutdown_triggers)
# add the node controller singleton session
#
self._controller = NodeControllerSession(self)
self._router_session_factory.add(self._controller, authrole=u'trusted')
# detect WAMPlets (FIXME: remove this!)
#
wamplets = self._controller._get_wamplets()
if len(wamplets) > 0:
self.log.info("Detected {wamplets} WAMPlets in environment:",
wamplets=len(wamplets))
for wpl in wamplets:
self.log.info("WAMPlet {dist}.{name}",
dist=wpl['dist'],
name=wpl['name'])
else:
self.log.debug("No WAMPlets detected in enviroment.")
panic = False
try:
# startup the node from local node configuration
#
yield self._startup(self._config)
# connect to CDC when running in managed mode
#
if cdc_mode:
cdc_config = controller_config.get(
'cdc',
{
# CDC connecting transport
u'transport': {
u'type': u'websocket',
u'url': u'wss://cdc.crossbario.com/ws',
u'endpoint': {
u'type': u'tcp',
u'host': u'cdc.crossbario.com',
u'port': 443,
u'timeout': 5,
u'tls': {
u'hostname': u'cdc.crossbario.com'
}
}
}
})
transport = cdc_config[u'transport']
hostname = None
if u'tls' in transport[u'endpoint']:
transport[u'endpoint'][u'tls'][u'hostname']
runner = ApplicationRunner(
url=transport['url'],
realm=None,
extra=None,
ssl=optionsForClientTLS(hostname) if hostname else None,
)
def make(config):
# extra info forwarded to CDC client session
extra = {
'node': self,
'on_ready': Deferred(),
'on_exit': Deferred(),
'node_key': self._node_key,
}
@inlineCallbacks
def on_ready(res):
self._manager, self._management_realm, self._node_id, self._node_extra = res
if self._bridge_session:
try:
yield self._bridge_session.attach_manager(
self._manager, self._management_realm,
self._node_id)
status = yield self._manager.call(
u'cdc.remote.status@1')
except:
self.log.failure()
else:
self.log.info(
'Connected to CDC for management realm "{realm}" (current time is {now})',
realm=self._management_realm,
now=status[u'now'])
else:
self.log.warn(
'Uplink CDC session established, but no bridge session setup!'
)
@inlineCallbacks
def on_exit(res):
if self._bridge_session:
try:
yield self._bridge_session.detach_manager()
except:
self.log.failure()
else:
self.log.info(
'Disconnected from CDC for management realm "{realm}"',
realm=self._management_realm)
else:
self.log.warn(
'Uplink CDC session lost, but no bridge session setup!'
)
self._manager, self._management_realm, self._node_id, self._node_extra = None, None, None, None
extra['on_ready'].addCallback(on_ready)
extra['on_exit'].addCallback(on_exit)
config = ComponentConfig(extra=extra)
session = NodeManagementSession(config)
return session
self.log.info("Connecting to CDC at '{url}' ..",
url=transport[u'url'])
yield runner.run(make,
start_reactor=False,
auto_reconnect=True)
# Notify systemd that crossbar is fully up and running
# (this has no effect on non-systemd platforms)
try:
import sdnotify
sdnotify.SystemdNotifier().notify("READY=1")
except:
pass
except ApplicationError as e:
panic = True
self.log.error("{msg}", msg=e.error_message())
except Exception:
panic = True
traceback.print_exc()
if panic:
try:
self._reactor.stop()
except twisted.internet.error.ReactorNotRunning:
pass
class Node(object):
"""
A Crossbar.io node is the running a controller process and one or multiple
worker processes.
A single Crossbar.io node runs exactly one instance of this class, hence
this class can be considered a system singleton.
"""
def __init__(self, reactor, options):
"""
Ctor.
:param reactor: Reactor to run on.
:type reactor: obj
:param options: Options from command line.
:type options: obj
"""
self.log = make_logger()
self.options = options
# the reactor under which we run
self._reactor = reactor
# shortname for reactor to run (when given via explicit option) or None
self._reactor_shortname = options.reactor
# node directory
self._cbdir = options.cbdir
# the node's name (must be unique within the management realm)
self._node_id = None
# the node's management realm
self._realm = None
# node controller session (a singleton ApplicationSession embedded
# in the node's management router)
self._controller = None
# config of this node.
self._config = None
def check_config(self):
"""
Check the configuration of this node.
"""
# for now, a node is always started from a local configuration
#
configfile = os.path.join(self.options.cbdir, self.options.config)
self.log.info("Loading node configuration file '{configfile}'",
configfile=configfile)
self._config = check_config_file(configfile, silence=True)
@inlineCallbacks
def start(self):
"""
Starts this node. This will start a node controller and then spawn new worker
processes as needed.
"""
if not self._config:
self.check_config()
controller_config = self._config.get('controller', {})
controller_options = controller_config.get('options', {})
controller_title = controller_options.get('title',
'crossbar-controller')
try:
import setproctitle
except ImportError:
self.log.warn(
"Warning, could not set process title (setproctitle not installed)"
)
else:
setproctitle.setproctitle(controller_title)
# the node's name (must be unique within the management realm)
if 'id' in controller_config:
self._node_id = controller_config['id']
else:
self._node_id = socket.gethostname()
if 'manager' in controller_config:
extra = {
'onready': Deferred(),
# authentication information for connecting to uplinkg CDC router
# using WAMP-CRA authentication
#
'authid': self._node_id,
'authkey': controller_config['manager']['key']
}
realm = controller_config['manager']['realm']
transport = controller_config['manager']['transport']
runner = ApplicationRunner(url=transport['url'],
realm=realm,
extra=extra,
debug_wamp=False)
runner.run(NodeManagementSession, start_reactor=False)
# wait until we have attached to the uplink CDC
self._management_session = yield extra['onready']
self.log.info(
"Node is connected to Crossbar.io DevOps Center (CDC)")
else:
self._management_session = None
# the node's management realm
self._realm = controller_config.get('realm', 'crossbar')
# router and factory that creates router sessions
#
self._router_factory = RouterFactory()
self._router_session_factory = RouterSessionFactory(
self._router_factory)
rlm = RouterRealm(None, {'name': self._realm})
# create a new router for the realm
router = self._router_factory.start_realm(rlm)
# add a router/realm service session
cfg = ComponentConfig(self._realm)
rlm.session = RouterServiceSession(cfg, router)
self._router_session_factory.add(rlm.session, authrole=u'trusted')
if self._management_session:
self._bridge_session = NodeManagementBridgeSession(
cfg, self._management_session)
self._router_session_factory.add(self._bridge_session,
authrole=u'trusted')
else:
self._bridge_session = None
# the node controller singleton WAMP application session
#
self._controller = NodeControllerSession(self)
# add the node controller singleton session to the router
#
self._router_session_factory.add(self._controller, authrole=u'trusted')
# Detect WAMPlets
#
wamplets = self._controller._get_wamplets()
if len(wamplets) > 0:
self.log.info("Detected {wamplets} WAMPlets in environment:",
wamplets=len(wamplets))
for wpl in wamplets:
self.log.info("WAMPlet {dist}.{name}",
dist=wpl['dist'],
name=wpl['name'])
else:
self.log.info("No WAMPlets detected in enviroment.")
panic = False
try:
yield self._startup(self._config)
except ApplicationError as e:
panic = True
for line in e.args[0].strip().splitlines():
self.log.error(line)
except Exception:
panic = True
traceback.print_exc()
if panic:
try:
self._reactor.stop()
except twisted.internet.error.ReactorNotRunning:
pass
@inlineCallbacks
def _startup(self, config):
# Setup node according to the local configuration provided.
# fake call details information when calling into
# remoted procedure locally
#
call_details = CallDetails(caller=0)
controller = config.get('controller', {})
# start Manhole in node controller
#
if 'manhole' in controller:
yield self._controller.start_manhole(controller['manhole'],
details=call_details)
# startup all workers
#
worker_no = 1
call_options = CallOptions(disclose_me=True)
for worker in config.get('workers', []):
# worker ID, type and logname
#
if 'id' in worker:
worker_id = worker.pop('id')
else:
worker_id = 'worker{}'.format(worker_no)
worker_no += 1
worker_type = worker['type']
worker_options = worker.get('options', {})
if worker_type == 'router':
worker_logname = "Router '{}'".format(worker_id)
elif worker_type == 'container':
worker_logname = "Container '{}'".format(worker_id)
elif worker_type == 'guest':
worker_logname = "Guest '{}'".format(worker_id)
else:
raise Exception("logic error")
# router/container
#
if worker_type in ['router', 'container']:
# start a new native worker process ..
#
if worker_type == 'router':
yield self._controller.start_router(worker_id,
worker_options,
details=call_details)
elif worker_type == 'container':
yield self._controller.start_container(
worker_id, worker_options, details=call_details)
else:
raise Exception("logic error")
# setup native worker generic stuff
#
if 'pythonpath' in worker_options:
added_paths = yield self._controller.call(
'crossbar.node.{}.worker.{}.add_pythonpath'.format(
self._node_id, worker_id),
worker_options['pythonpath'],
options=call_options)
self.log.debug("{worker}: PYTHONPATH extended for {paths}",
worker=worker_logname,
paths=added_paths)
if 'cpu_affinity' in worker_options:
new_affinity = yield self._controller.call(
'crossbar.node.{}.worker.{}.set_cpu_affinity'.format(
self._node_id, worker_id),
worker_options['cpu_affinity'],
options=call_options)
self.log.debug("{worker}: CPU affinity set to {affinity}",
worker=worker_logname,
affinity=new_affinity)
if 'manhole' in worker:
yield self._controller.call(
'crossbar.node.{}.worker.{}.start_manhole'.format(
self._node_id, worker_id),
worker['manhole'],
options=call_options)
self.log.debug("{worker}: manhole started",
worker=worker_logname)
# setup router worker
#
if worker_type == 'router':
# start realms on router
#
realm_no = 1
for realm in worker.get('realms', []):
if 'id' in realm:
realm_id = realm.pop('id')
else:
realm_id = 'realm{}'.format(realm_no)
realm_no += 1
# extract schema information from WAMP-flavored Markdown
#
schemas = None
if 'schemas' in realm:
schemas = {}
schema_pat = re.compile(r"```javascript(.*?)```",
re.DOTALL)
cnt_files = 0
cnt_decls = 0
for schema_file in realm.pop('schemas'):
schema_file = os.path.join(
self.options.cbdir, schema_file)
self.log.info(
"{worker}: processing WAMP-flavored Markdown file {schema_file} for WAMP schema declarations",
worker=worker_logname,
schema_file=schema_file)
with open(schema_file, 'r') as f:
cnt_files += 1
for d in schema_pat.findall(f.read()):
try:
o = json.loads(d)
if isinstance(
o, dict
) and '$schema' in o and o[
'$schema'] == u'http://wamp.ws/schema#':
uri = o['uri']
if uri not in schemas:
schemas[uri] = {}
schemas[uri].update(o)
cnt_decls += 1
except Exception:
self.log.failure(
"{worker}: WARNING - failed to process declaration in {schema_file} - {log_failure.value}",
worker=worker_logname,
schema_file=schema_file)
self.log.info(
"{worker}: processed {cnt_files} files extracting {cnt_decls} schema declarations and {len_schemas} URIs",
worker=worker_logname,
cnt_files=cnt_files,
cnt_decls=cnt_decls,
len_schemas=len(schemas))
yield self._controller.call(
'crossbar.node.{}.worker.{}.start_router_realm'.
format(self._node_id, worker_id),
realm_id,
realm,
schemas,
options=call_options)
self.log.info(
"{worker}: realm '{realm_id}' (named '{realm_name}') started",
worker=worker_logname,
realm_id=realm_id,
realm_name=realm['name'])
# add roles to realm
#
role_no = 1
for role in realm.get('roles', []):
if 'id' in role:
role_id = role.pop('id')
else:
role_id = 'role{}'.format(role_no)
role_no += 1
yield self._controller.call(
'crossbar.node.{}.worker.{}.start_router_realm_role'
.format(self._node_id, worker_id),
realm_id,
role_id,
role,
options=call_options)
self.log.info(
"{}: role '{}' (named '{}') started on realm '{}'"
.format(worker_logname, role_id, role['name'],
realm_id))
# start connections (such as PostgreSQL database connection pools)
# to run embedded in the router
#
connection_no = 1
for connection in worker.get('connections', []):
if 'id' in connection:
connection_id = connection.pop('id')
else:
connection_id = 'connection{}'.format(
connection_no)
connection_no += 1
yield self._controller.call(
'crossbar.node.{}.worker.{}.start_connection'.
format(self._node_id, worker_id),
connection_id,
connection,
options=call_options)
self.log.info("{}: connection '{}' started".format(
worker_logname, connection_id))
# start components to run embedded in the router
#
component_no = 1
for component in worker.get('components', []):
if 'id' in component:
component_id = component.pop('id')
else:
component_id = 'component{}'.format(component_no)
component_no += 1
yield self._controller.call(
'crossbar.node.{}.worker.{}.start_router_component'
.format(self._node_id, worker_id),
component_id,
component,
options=call_options)
self.log.info("{}: component '{}' started".format(
worker_logname, component_id))
# start transports on router
#
transport_no = 1
for transport in worker['transports']:
if 'id' in transport:
transport_id = transport.pop('id')
else:
transport_id = 'transport{}'.format(transport_no)
transport_no += 1
yield self._controller.call(
'crossbar.node.{}.worker.{}.start_router_transport'
.format(self._node_id, worker_id),
transport_id,
transport,
options=call_options)
self.log.info("{}: transport '{}' started".format(
worker_logname, transport_id))
# setup container worker
#
elif worker_type == 'container':
component_no = 1
# if components exit "very soon after" we try to
# start them, we consider that a failure and shut
# our node down. We remove this subscription 2
# seconds after we're done starting everything
# (see below). This is necessary as
# start_container_component returns as soon as
# we've established a connection to the component
def component_exited(info):
component_id = info.id if hasattr(info, 'id') else None
self.log.info(
"Component '{component_id}' failed to start; shutting down node.",
component_id=component_id)
try:
self._reactor.stop()
except twisted.internet.error.ReactorNotRunning:
pass
topic = 'crossbar.node.{}.worker.{}.container.on_component_stop'.format(
self._node_id, worker_id)
component_stop_sub = yield self._controller.subscribe(
component_exited, topic)
# start connections (such as PostgreSQL database connection pools)
# to run embedded in the container
#
connection_no = 1
for connection in worker.get('connections', []):
if 'id' in connection:
connection_id = connection.pop('id')
else:
connection_id = 'connection{}'.format(
connection_no)
connection_no += 1
yield self._controller.call(
'crossbar.node.{}.worker.{}.start_connection'.
format(self._node_id, worker_id),
connection_id,
connection,
options=call_options)
self.log.info("{}: connection '{}' started".format(
worker_logname, connection_id))
# start components to run embedded in the container
#
for component in worker.get('components', []):
if 'id' in component:
component_id = component.pop('id')
else:
component_id = 'component{}'.format(component_no)
component_no += 1
yield self._controller.call(
'crossbar.node.{}.worker.{}.start_container_component'
.format(self._node_id, worker_id),
component_id,
component,
options=call_options)
self.log.info(
"{worker}: component '{component_id}' started",
worker=worker_logname,
component_id=component_id)
# after 2 seconds, consider all the application components running
self._reactor.callLater(2, component_stop_sub.unsubscribe)
else:
raise Exception("logic error")
elif worker_type == 'guest':
# start guest worker
#
yield self._controller.start_guest(worker_id,
worker,
details=call_details)
self.log.info("{worker}: started", worker=worker_logname)
else:
raise Exception("logic error")
def start(self):
"""
Starts this node. This will start a node controller and then spawn new worker
processes as needed.
"""
# for now, a node is always started from a local configuration
#
configfile = os.path.join(self.options.cbdir, self.options.config)
self.log.info("Starting from node configuration file '{configfile}'",
configfile=configfile)
self._config = check_config_file(configfile, silence=True)
controller_config = self._config.get('controller', {})
controller_options = controller_config.get('options', {})
controller_title = controller_options.get('title', 'crossbar-controller')
try:
import setproctitle
except ImportError:
self.log.warn("Warning, could not set process title (setproctitle not installed)")
else:
setproctitle.setproctitle(controller_title)
# the node's name (must be unique within the management realm)
if 'manager' in self._config:
self._node_id = self._config['manager']['id']
else:
if 'id' in controller_config:
self._node_id = controller_config['id']
else:
self._node_id = socket.gethostname()
if 'manager' in self._config:
extra = {
'onready': Deferred()
}
runner = ApplicationRunner(url=u"ws://localhost:9000", realm=u"cdc-oberstet-1", extra=extra)
runner.run(NodeManagementSession, start_reactor=False)
# wait until we have attached to the uplink CDC
self._management_session = yield extra['onready']
self.log.info("Connected to Crossbar.io Management Cloud: {management_session}",
management_session=self._management_session)
else:
self._management_session = None
# the node's management realm
self._realm = controller_config.get('realm', 'crossbar')
# router and factory that creates router sessions
#
self._router_factory = RouterFactory()
self._router_session_factory = RouterSessionFactory(self._router_factory)
rlm = RouterRealm(None, {'name': self._realm})
# create a new router for the realm
router = self._router_factory.start_realm(rlm)
# add a router/realm service session
cfg = ComponentConfig(self._realm)
rlm.session = RouterServiceSession(cfg, router)
self._router_session_factory.add(rlm.session, authrole=u'trusted')
if self._management_session:
self._bridge_session = NodeManagementBridgeSession(cfg, self._management_session)
self._router_session_factory.add(self._bridge_session, authrole=u'trusted')
else:
self._bridge_session = None
# the node controller singleton WAMP application session
#
# session_config = ComponentConfig(realm = options.realm, extra = options)
self._controller = NodeControllerSession(self)
# add the node controller singleton session to the router
#
self._router_session_factory.add(self._controller, authrole=u'trusted')
# Detect WAMPlets
#
wamplets = self._controller._get_wamplets()
if len(wamplets) > 0:
self.log.info("Detected {wamplets} WAMPlets in environment:",
wamplets=len(wamplets))
for wpl in wamplets:
self.log.info("WAMPlet {dist}.{name}",
dist=wpl['dist'], name=wpl['name'])
else:
self.log.info("No WAMPlets detected in enviroment.")
try:
if 'manager' in self._config:
yield self._startup_managed(self._config)
else:
yield self._startup_standalone(self._config)
except:
traceback.print_exc()
try:
self._reactor.stop()
except twisted.internet.error.ReactorNotRunning:
pass
class Node:
"""
A Crossbar.io node is the running a controller process
and one or multiple worker processes.
A single Crossbar.io node runs exactly one instance of
this class, hence this class can be considered a system
singleton.
"""
def __init__(self, reactor, options):
"""
Ctor.
:param reactor: Reactor to run on.
:type reactor: obj
:param options: Options from command line.
:type options: obj
"""
self.debug = False
self.options = options
## the reactor under which we run
self._reactor = reactor
## shortname for reactor to run (when given via explicit option) or None
self._reactor_shortname = options.reactor
## node directory
self._cbdir = options.cbdir
## the node's name (must be unique within the management realm)
self._node_id = None
## the node's management realm
self._realm = None
## node controller session (a singleton ApplicationSession embedded
## in the node's management router)
self._controller = None
def start(self):
"""
Starts this node. This will start a node controller and then spawn new worker
processes as needed.
"""
## for now, a node is always started from a local configuration
##
configfile = os.path.join(self.options.cbdir, self.options.config)
log.msg("Starting from local configuration '{}'".format(configfile))
config = checkconfig.check_config_file(configfile, silence = True)
self.start_from_config(config)
def start_from_config(self, config):
controller_config = config.get('controller', {})
controller_options = controller_config.get('options', {})
controller_title = controller_options.get('title', 'crossbar-controller')
try:
import setproctitle
except ImportError:
log.msg("Warning, could not set process title (setproctitle not installed)")
else:
setproctitle.setproctitle(controller_title)
## the node's name (must be unique within the management realm)
if 'id' in controller_config:
self._node_id = controller_config['id']
else:
self._node_id = socket.gethostname()
## the node's management realm
self._realm = controller_config.get('realm', 'crossbar')
## the node controller singleton WAMP application session
##
#session_config = ComponentConfig(realm = options.realm, extra = options)
self._controller = NodeControllerSession(self)
## router and factory that creates router sessions
##
self._router_factory = RouterFactory(
options = wamp.types.RouterOptions(uri_check = wamp.types.RouterOptions.URI_CHECK_LOOSE),
debug = False)
self._router_session_factory = RouterSessionFactory(self._router_factory)
## add the node controller singleton session to the router
##
self._router_session_factory.add(self._controller)
## Detect WAMPlets
##
wamplets = self._controller._get_wamplets()
if len(wamplets) > 0:
log.msg("Detected {} WAMPlets in environment:".format(len(wamplets)))
for wpl in wamplets:
log.msg("WAMPlet {}.{}".format(wpl['dist'], wpl['name']))
else:
log.msg("No WAMPlets detected in enviroment.")
self.run_node_config(config)
def _start_from_local_config(self, configfile):
"""
Start Crossbar.io node from local configuration file.
"""
configfile = os.path.abspath(configfile)
log.msg("Starting from local config file '{}'".format(configfile))
try:
config = controller.config.check_config_file(configfile, silence = True)
#config = json.loads(open(configfile, 'rb').read())
except Exception as e:
log.msg("Fatal: {}".format(e))
sys.exit(1)
else:
self.run_node_config(config)
@inlineCallbacks
def run_node_config(self, config):
try:
yield self._run_node_config(config)
except:
traceback.print_exc()
self._reactor.stop()
@inlineCallbacks
def _run_node_config(self, config):
"""
Setup node according to config provided.
"""
## fake call details information when calling into
## remoted procedure locally
##
call_details = CallDetails(caller = 0, authid = 'node')
controller = config.get('controller', {})
## start Manhole in node controller
##
if 'manhole' in controller:
yield self._controller.start_manhole(controller['manhole'], details = call_details)
## start local transport for management router
##
if 'transport' in controller:
yield self._controller.start_management_transport(controller['transport'], details = call_details)
## startup all workers
##
worker_no = 1
for worker in config.get('workers', []):
## worker ID, type and logname
##
if 'id' in worker:
worker_id = worker.pop('id')
else:
worker_id = 'worker{}'.format(worker_no)
worker_no += 1
worker_type = worker['type']
worker_options = worker.get('options', {})
if worker_type == 'router':
worker_logname = "Router '{}'".format(worker_id)
elif worker_type == 'container':
worker_logname = "Container '{}'".format(worker_id)
elif worker_type == 'guest':
worker_logname = "Guest '{}'".format(worker_id)
else:
raise Exception("logic error")
## router/container
##
if worker_type in ['router', 'container']:
## start a new native worker process ..
##
if worker_type == 'router':
yield self._controller.start_router(worker_id, worker_options, details = call_details)
elif worker_type == 'container':
yield self._controller.start_container(worker_id, worker_options, details = call_details)
else:
raise Exception("logic error")
## setup native worker generic stuff
##
if 'pythonpath' in worker_options:
added_paths = yield self._controller.call('crossbar.node.{}.worker.{}.add_pythonpath'.format(self._node_id, worker_id), worker_options['pythonpath'])
if self.debug:
log.msg("{}: PYTHONPATH extended for {}".format(worker_logname, added_paths))
else:
log.msg("{}: PYTHONPATH extended".format(worker_logname))
if 'cpu_affinity' in worker_options:
new_affinity = yield self._controller.call('crossbar.node.{}.worker.{}.set_cpu_affinity'.format(self._node_id, worker_id), worker_options['cpu_affinity'])
log.msg("{}: CPU affinity set to {}".format(worker_logname, new_affinity))
if 'manhole' in worker:
yield self._controller.call('crossbar.node.{}.worker.{}.start_manhole'.format(self._node_id, worker_id), worker['manhole'])
log.msg("{}: manhole started".format(worker_logname))
## setup router worker
##
if worker_type == 'router':
## start realms on router
##
realm_no = 1
for realm in worker.get('realms', []):
if 'id' in realm:
realm_id = realm.pop('id')
else:
realm_id = 'realm{}'.format(realm_no)
realm_no += 1
## FIXME
#yield self._controller.call('crossbar.node.{}.worker.{}.start_router_realm'.format(self._node_id, worker_id), realm_id, realm)
#log.msg("{}: realm '{}' started".format(worker_logname, realm_id))
## start components to run embedded in the router
##
component_no = 1
for component in worker.get('components', []):
if 'id' in component:
component_id = component.pop('id')
else:
component_id = 'component{}'.format(component_no)
component_no += 1
yield self._controller.call('crossbar.node.{}.worker.{}.start_router_component'.format(self._node_id, worker_id), component_id, component)
log.msg("{}: component '{}' started".format(worker_logname, component_id))
## start transports on router
##
transport_no = 1
for transport in worker['transports']:
if 'id' in transport:
transport_id = transport.pop('id')
else:
transport_id = 'transport{}'.format(transport_no)
transport_no += 1
yield self._controller.call('crossbar.node.{}.worker.{}.start_router_transport'.format(self._node_id, worker_id), transport_id, transport)
log.msg("{}: transport '{}' started".format(worker_logname, transport_id))
## setup container worker
##
elif worker_type == 'container':
component_no = 1
for component in worker.get('components', []):
if 'id' in component:
component_id = component.pop('id')
else:
component_id = 'component{}'.format(component_no)
component_no += 1
yield self._controller.call('crossbar.node.{}.worker.{}.start_container_component'.format(self._node_id, worker_id), component_id, component)
log.msg("{}: component '{}' started".format(worker_logname, component_id))
else:
raise Exception("logic error")
elif worker_type == 'guest':
## start guest worker
##
yield self._controller.start_guest(worker_id, worker, details = call_details)
log.msg("{}: started".format(worker_logname))
else:
raise Exception("logic error")