def publish_started(session, start_details):
self.log.info(
"started component: {session} id={session_id}",
session=class_name(session),
session_id=session._session_id,
)
topic = self._uri_prefix + '.container.on_component_start'
event = {u'id': id}
caller = details.caller if details else None
self.publish(topic, event, options=PublishOptions(exclude=caller))
return event
def publish_started(session, start_details):
self.log.info(
"started component: {session} id={session_id}",
session=class_name(session),
session_id=session._session_id,
)
topic = self._uri_prefix + '.on_component_start'
event = {u'id': id}
caller = details.caller if details else None
self.publish(topic, event, options=PublishOptions(exclude=caller))
return event
def publish_stopped(session, stop_details):
self.log.info(
"stopped component: {session} id={session_id}",
session=class_name(session),
session_id=session._session_id,
)
topic = self._uri_prefix + '.on_component_stop'
event = {'id': id}
caller = details.caller if details else None
self.publish(topic, event, options=PublishOptions(exclude=caller))
if not started_d.called:
started_d.errback(Exception("Session left before being ready"))
return event
def publish_ready(session):
"""
when our component is ready, we publish .on_component_ready
"""
self.log.info(
"component ready: {session} id={session_id}",
session=class_name(session),
session_id=session._session_id,
)
topic = self._uri_prefix + '.on_component_ready'
event = {'id': id}
self.publish(topic, event)
started_d.callback(event)
return event
def publish_started(session, start_details):
"""
when our component starts, we publish .on_component_start
"""
# hook up handlers for "session is ready"
session.on('ready', publish_ready)
# publish .on_component_start
self.log.info(
"started component: {session} id={session_id}",
session=class_name(session),
session_id=session._session_id,
)
topic = self._uri_prefix + '.on_component_start'
event = {'id': id}
caller = details.caller if details else None
self.publish(topic, event, options=PublishOptions(exclude=caller))
return event
def start_router_component(self, id, config, details=None):
"""
Start an app component in this router worker.
:param id: The ID of the component to start.
:type id: str
:param config: The component configuration.
:type config: dict
:param details: Call details.
:type details: :class:`autobahn.wamp.types.CallDetails`
"""
self.log.debug("{name}.start_router_component",
name=self.__class__.__name__)
# prohibit starting a component twice
#
if id in self.components:
emsg = "Could not start component: a component with ID '{}'' is already running (or starting)".format(
id)
self.log.error(emsg)
raise ApplicationError(u'crossbar.error.already_running', emsg)
# check configuration
#
try:
self.personality.check_router_component(self.personality, config)
except Exception as e:
emsg = "Invalid router component configuration: {}".format(e)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.invalid_configuration",
emsg)
else:
self.log.debug("Starting {type}-component on router.",
type=config['type'])
# resolve references to other entities
#
references = {}
for ref in config.get('references', []):
ref_type, ref_id = ref.split(':')
if ref_type == u'connection':
if ref_id in self._connections:
references[ref] = self._connections[ref_id]
else:
emsg = "cannot resolve reference '{}' - no '{}' with ID '{}'".format(
ref, ref_type, ref_id)
self.log.error(emsg)
raise ApplicationError(
u"crossbar.error.invalid_configuration", emsg)
else:
emsg = "cannot resolve reference '{}' - invalid reference type '{}'".format(
ref, ref_type)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.invalid_configuration",
emsg)
# create component config
#
realm = config['realm']
extra = config.get('extra', None)
component_config = ComponentConfig(
realm=realm,
extra=extra,
keyring=None,
controller=self if self.config.extra.expose_controller else None,
shared=self.components_shared
if self.config.extra.expose_shared else None)
create_component = _appsession_loader(config)
# .. and create and add an WAMP application session to
# run the component next to the router
#
try:
session = create_component(component_config)
# any exception spilling out from user code in onXXX handlers is fatal!
def panic(fail, msg):
self.log.error(
"Fatal error in component: {msg} - {log_failure.value}",
msg=msg,
log_failure=fail)
session.disconnect()
session._swallow_error = panic
except Exception:
self.log.error(
"Component instantiation failed",
log_failure=Failure(),
)
raise
# Note that 'join' is fired to listeners *before* onJoin runs,
# so if you do 'yield self.leave()' in onJoin we'll still
# publish "started" before "stopped".
def publish_stopped(session, stop_details):
self.log.info(
"stopped component: {session} id={session_id}",
session=class_name(session),
session_id=session._session_id,
)
topic = self._uri_prefix + '.on_component_stop'
event = {u'id': id}
caller = details.caller if details else None
self.publish(topic, event, options=PublishOptions(exclude=caller))
return event
def publish_started(session, start_details):
self.log.info(
"started component: {session} id={session_id}",
session=class_name(session),
session_id=session._session_id,
)
topic = self._uri_prefix + '.on_component_start'
event = {u'id': id}
caller = details.caller if details else None
self.publish(topic, event, options=PublishOptions(exclude=caller))
return event
session.on('leave', publish_stopped)
session.on('join', publish_started)
self.components[id] = RouterComponent(id, config, session)
self._router_session_factory.add(session,
authrole=config.get(
'role', u'anonymous'))
self.log.debug(
"Added component {id} (type '{name}')",
id=id,
name=class_name(session),
)
def start_router_component(self, id, config, details=None):
"""
Start an app component in this router worker.
:param id: The ID of the component to start.
:type id: str
:param config: The component configuration.
:type config: dict
:param details: Call details.
:type details: :class:`autobahn.wamp.types.CallDetails`
"""
self.log.debug("{name}.start_router_component", name=self.__class__.__name__)
# prohibit starting a component twice
#
if id in self.components:
emsg = "Could not start component: a component with ID '{}'' is already running (or starting)".format(id)
self.log.error(emsg)
raise ApplicationError('crossbar.error.already_running', emsg)
started_d = Deferred()
# check configuration
#
try:
self.personality.check_router_component(self.personality, config)
except Exception as e:
emsg = "Invalid router component configuration: {}".format(e)
self.log.error(emsg)
raise ApplicationError("crossbar.error.invalid_configuration", emsg)
else:
self.log.debug("Starting {type}-component on router.",
type=config['type'])
# resolve references to other entities
#
references = {}
for ref in config.get('references', []):
ref_type, ref_id = ref.split(':')
if ref_type == 'connection':
if ref_id in self._connections:
references[ref] = self._connections[ref_id]
else:
emsg = "cannot resolve reference '{}' - no '{}' with ID '{}'".format(ref, ref_type, ref_id)
self.log.error(emsg)
raise ApplicationError("crossbar.error.invalid_configuration", emsg)
else:
emsg = "cannot resolve reference '{}' - invalid reference type '{}'".format(ref, ref_type)
self.log.error(emsg)
raise ApplicationError("crossbar.error.invalid_configuration", emsg)
# create component config
#
realm = config.get('realm', None)
assert isinstance(realm, str)
extra = config.get('extra', {})
assert isinstance(extra, dict)
# forward crossbar node base directory
extra['cbdir'] = self.config.extra.cbdir
# allow access to controller session
controller = self if self.config.extra.expose_controller else None
# expose an object shared between components
shared = self.components_shared if self.config.extra.expose_shared else None
# this is the component configuration provided to the components ApplicationSession
component_config = ComponentConfig(realm=realm,
extra=extra,
keyring=None,
controller=controller,
shared=shared)
# define component ctor function
try:
create_component = _appsession_loader(config)
except ApplicationError as e:
# for convenience, also log failed component loading
self.log.error('component loading failed', log_failure=Failure())
if 'No module named' in str(e):
self.log.error(' Python module search paths:')
for path in e.kwargs['pythonpath']:
self.log.error(' {path}', path=path)
raise
# .. and create and add an WAMP application session to
# run the component next to the router
try:
session = create_component(component_config)
# any exception spilling out from user code in onXXX handlers is fatal!
def panic(fail, msg):
self.log.error(
"Fatal error in component: {msg} - {log_failure.value}",
msg=msg, log_failure=fail
)
session.disconnect()
session._swallow_error = panic
except Exception:
self.log.error(
"Component instantiation failed",
log_failure=Failure(),
)
raise
# Note that 'join' is fired to listeners *before* onJoin runs,
# so if you do 'yield self.leave()' in onJoin we'll still
# publish "started" before "stopped".
def publish_stopped(session, stop_details):
self.log.info(
"stopped component: {session} id={session_id}",
session=class_name(session),
session_id=session._session_id,
)
topic = self._uri_prefix + '.on_component_stop'
event = {'id': id}
caller = details.caller if details else None
self.publish(topic, event, options=PublishOptions(exclude=caller))
if not started_d.called:
started_d.errback(Exception("Session left before being ready"))
return event
def publish_ready(session):
"""
when our component is ready, we publish .on_component_ready
"""
self.log.info(
"component ready: {session} id={session_id}",
session=class_name(session),
session_id=session._session_id,
)
topic = self._uri_prefix + '.on_component_ready'
event = {'id': id}
self.publish(topic, event)
started_d.callback(event)
return event
def publish_started(session, start_details):
"""
when our component starts, we publish .on_component_start
"""
# hook up handlers for "session is ready"
session.on('ready', publish_ready)
# publish .on_component_start
self.log.info(
"started component: {session} id={session_id}",
session=class_name(session),
session_id=session._session_id,
)
topic = self._uri_prefix + '.on_component_start'
event = {'id': id}
caller = details.caller if details else None
self.publish(topic, event, options=PublishOptions(exclude=caller))
return event
session.on('leave', publish_stopped)
session.on('join', publish_started)
self.components[id] = RouterComponent(id, config, session)
router = self._router_factory.get(realm)
self._router_session_factory.add(session, router, authrole=config.get('role', 'anonymous'))
self.log.debug(
"Added component {id} (type '{name}')",
id=id,
name=class_name(session),
)
return started_d
def _start_native_worker(self,
worker_type,
worker_id,
worker_options=None,
details=None):
# prohibit starting a worker twice
#
if worker_id in self._workers:
emsg = "Could not start worker: a worker with ID '{}' is already running (or starting)".format(
worker_id)
self.log.error(emsg)
raise ApplicationError(u'crossbar.error.worker_already_running',
emsg)
# check worker options
#
options = worker_options or {}
try:
if worker_type in self._node._native_workers:
if self._node._native_workers[worker_type][
'checkconfig_options']:
self._node._native_workers[worker_type][
'checkconfig_options'](self.personality, options)
else:
raise Exception(
'No checkconfig_options for worker type "{worker_type}" implemented!'
.format(worker_type=worker_type))
else:
raise Exception('invalid worker type "{}"'.format(worker_type))
except Exception as e:
emsg = "Could not start native worker: invalid configuration ({})".format(
e)
self.log.error(emsg)
raise ApplicationError(u'crossbar.error.invalid_configuration',
emsg)
# the fully qualified worker class as a string
worker_class = qual(
self._node._native_workers[worker_type]['worker_class'])
# allow override Python executable from options
#
if 'python' in options:
exe = options['python']
# the executable must be an absolute path, e.g. /home/oberstet/pypy-2.2.1-linux64/bin/pypy
#
if not os.path.isabs(exe):
emsg = "Invalid worker configuration: python executable '{}' must be an absolute path".format(
exe)
self.log.error(emsg)
raise ApplicationError(u'crossbar.error.invalid_configuration',
emsg)
# of course the path must exist and actually be executable
#
if not (os.path.isfile(exe) and os.access(exe, os.X_OK)):
emsg = "Invalid worker configuration: python executable '{}' does not exist or isn't an executable".format(
exe)
self.log.error(emsg)
raise ApplicationError(u'crossbar.error.invalid_configuration',
emsg)
else:
exe = sys.executable
# allow override default Python module search paths from options
#
if 'pythonpath' in options:
pythonpaths_to_add = [
os.path.abspath(os.path.join(self._node._cbdir, p))
for p in options.get('pythonpath', [])
]
else:
pythonpaths_to_add = []
# assemble command line for forking the worker
#
# all native workers (routers and containers for now) start
# from the same script in crossbar/worker/process.py or
# from the command "crossbar _exec_worker" when crossbar is
# running from a frozen executable (single-file, pyinstaller, etc)
#
if getattr(sys, 'frozen', False):
# if we are inside a frozen crossbar executable, we need to invoke
# the crossbar executable with a command ("_exec_worker")
args = [exe, self._node.personality.NAME, "_exec_worker"]
else:
# we are invoking via "-m" so that .pyc files, __pycache__
# etc work properly. this works everywhere, but frozen executables
args = [exe, "-u", "-m", "crossbar.worker.main"]
args.extend(["--cbdir", self._node._cbdir])
args.extend(["--node", str(self._node._node_id)])
args.extend(["--worker", str(worker_id)])
args.extend(["--realm", self._realm])
args.extend(["--personality", class_name(self._node.personality)])
args.extend(["--klass", worker_class])
args.extend(["--loglevel", get_global_log_level()])
if self._node.options.debug_lifecycle:
args.append("--debug-lifecycle")
if self._node.options.debug_programflow:
args.append("--debug-programflow")
if self._node.options.enable_vmprof:
args.append("--vmprof")
if "shutdown" in options:
args.extend(["--shutdown", options["shutdown"]])
# Node-level callback to inject worker arguments
#
self._node._extend_worker_args(args, options)
# allow override worker process title from options
#
if options.get('title', None):
args.extend(['--title', options['title']])
# forward explicit reactor selection
#
if 'reactor' in options and sys.platform in options['reactor']:
args.extend(['--reactor', options['reactor'][sys.platform]])
# FIXME
# elif self._node.options.reactor:
# args.extend(['--reactor', self._node.options.reactor])
# create worker process environment
#
worker_env = create_process_env(options)
# We need to use the same PYTHONPATH we were started with, so we can
# find the Crossbar we're working with -- it may not be the same as the
# one on the default path
worker_env["PYTHONPATH"] = os.pathsep.join(pythonpaths_to_add +
sys.path)
# log name of worker
#
worker_logname = self._node._native_workers[worker_type]['logname']
# each worker is run under its own dedicated WAMP auth role
#
worker_auth_role = u'crossbar.worker.{}'.format(worker_id)
# topic URIs used (later)
#
starting_topic = self._node._native_workers[worker_type]['topics'][
'starting']
started_topic = self._node._native_workers[worker_type]['topics'][
'started']
# add worker tracking instance to the worker map ..
#
WORKER = self._node._native_workers[worker_type]['class']
worker = WORKER(self,
worker_id,
details.caller,
keeplog=options.get('traceback', None))
self._workers[worker_id] = worker
# create a (custom) process endpoint.
#
if platform.isWindows():
childFDs = None # Use the default Twisted ones
else:
# The communication between controller and container workers is
# using WAMP running over 2 pipes.
# For controller->native-worker traffic this runs over FD 0 (`stdin`)
# and for the native-worker->controller traffic, this runs over FD 3.
#
# Note: We use FD 3, not FD 1 (`stdout`) or FD 2 (`stderr`) for
# container->controller traffic, so that components running in the
# container which happen to write to `stdout` or `stderr` do not
# interfere with the container-controller communication.
childFDs = {0: "w", 1: "r", 2: "r", 3: "r"}
ep = WorkerProcessEndpoint(self._node._reactor,
exe,
args,
env=worker_env,
worker=worker,
childFDs=childFDs)
# ready handling
#
def on_ready_success(worker_id):
self.log.debug(
'{worker_type} worker "{worker_id}" process {pid} started',
worker_type=worker_logname,
worker_id=worker.id,
pid=worker.pid)
self._node._reactor.addSystemEventTrigger(
'before',
'shutdown',
self._cleanup_worker,
self._node._reactor,
worker,
)
worker.on_worker_started()
started_info = {
u'id':
worker.id,
u'status':
worker.status,
u'started':
utcstr(worker.started),
u'who':
worker.who,
u'pid':
worker.pid,
u'startup_time':
(worker.started -
worker.created).total_seconds() if worker.started else None
}
# FIXME: make start of stats printer dependent on log level ..
if False:
worker.log_stats(5.)
self.publish(started_topic,
started_info,
options=PublishOptions(exclude=details.caller))
return started_info
def on_ready_error(err):
del self._workers[worker.id]
emsg = 'Failed to start native worker: {}'.format(err.value)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.cannot_start", emsg,
worker.getlog())
worker.ready.addCallbacks(on_ready_success, on_ready_error)
def on_exit_success(_):
self.log.info("Node worker {worker.id} ended successfully",
worker=worker)
# clear worker log
worker.log_stats(0)
# remove the dedicated node router authrole we dynamically
# added for the worker
self._node._drop_worker_role(worker_auth_role)
# remove our metadata tracking for the worker
del self._workers[worker.id]
# indicate that the worker excited successfully
return True
def on_exit_error(err):
self.log.info("Node worker {worker.id} ended with error ({err})",
worker=worker,
err=err)
# clear worker log
worker.log_stats(0)
# remove the dedicated node router authrole we dynamically
# added for the worker
self._node._drop_worker_role(worker_auth_role)
# remove our metadata tracking for the worker
del self._workers[worker.id]
# indicate that the worker excited with error
return False
def check_for_shutdown(was_successful):
self.log.info(
'Checking for node shutdown: worker_exit_success={worker_exit_success}, shutdown_requested={shutdown_requested}, node_shutdown_triggers={node_shutdown_triggers}',
worker_exit_success=was_successful,
shutdown_requested=self._shutdown_requested,
node_shutdown_triggers=self._node._node_shutdown_triggers)
shutdown = self._shutdown_requested
# automatically shutdown node whenever a worker ended (successfully, or with error)
#
if NODE_SHUTDOWN_ON_WORKER_EXIT in self._node._node_shutdown_triggers:
self.log.info(
"Node worker ended, and trigger '{trigger}' is active: will shutdown node ..",
trigger=NODE_SHUTDOWN_ON_WORKER_EXIT)
term_print('CROSSBAR:NODE_SHUTDOWN_ON_WORKER_EXIT')
shutdown = True
# automatically shutdown node when worker ended with error
#
elif not was_successful and NODE_SHUTDOWN_ON_WORKER_EXIT_WITH_ERROR in self._node._node_shutdown_triggers:
self.log.info(
"Node worker ended with error, and trigger '{trigger}' is active: will shutdown node ..",
trigger=NODE_SHUTDOWN_ON_WORKER_EXIT_WITH_ERROR)
term_print('CROSSBAR:NODE_SHUTDOWN_ON_WORKER_EXIT_WITH_ERROR')
shutdown = True
# automatically shutdown node when no more workers are left
#
elif len(
self._workers
) == 0 and NODE_SHUTDOWN_ON_LAST_WORKER_EXIT in self._node._node_shutdown_triggers:
self.log.info(
"No more node workers running, and trigger '{trigger}' is active: will shutdown node ..",
trigger=NODE_SHUTDOWN_ON_LAST_WORKER_EXIT)
term_print('CROSSBAR:NODE_SHUTDOWN_ON_LAST_WORKER_EXIT')
shutdown = True
# initiate shutdown (but only if we are not already shutting down)
#
if shutdown:
self.shutdown()
else:
self.log.info('Node will continue to run!')
d_on_exit = worker.exit.addCallbacks(on_exit_success, on_exit_error)
d_on_exit.addBoth(check_for_shutdown)
# create a transport factory for talking WAMP to the native worker
#
transport_factory = create_native_worker_client_factory(
self._node._router_session_factory, worker_auth_role, worker.ready,
worker.exit)
transport_factory.noisy = False
self._workers[worker_id].factory = transport_factory
# now (immediately before actually forking) signal the starting of the worker
#
starting_info = {
u'id': worker_id,
u'status': worker.status,
u'created': utcstr(worker.created),
u'who': worker.who,
}
# the caller gets a progressive result ..
if details.progress:
details.progress(starting_info)
# .. while all others get an event
self.publish(starting_topic,
starting_info,
options=PublishOptions(exclude=details.caller))
# only the following line will actually exec a new worker process - everything before is just setup
# for this moment:
self.log.debug(
'Starting new managed worker process for {worker_logname} worker "{worker_id}" using {exe} with args {args}',
worker_id=worker_id,
worker_logname=worker_logname,
exe=exe,
args=args)
d = ep.connect(transport_factory)
def on_connect_success(proto):
# this seems to be called immediately when the child process
# has been forked. even if it then immediately fails because
# e.g. the executable doesn't even exist. in other words,
# I'm not sure under what conditions the deferred will errback ..
self.log.debug('Native worker "{worker_id}" connected',
worker_id=worker_id)
worker.on_worker_connected(proto)
# dynamically add a dedicated authrole to the router
# for the worker we've just started
self._node._add_worker_role(worker_auth_role, options)
def on_connect_error(err):
# not sure when this errback is triggered at all ..
self.log.error(
"Internal error: connection to forked native worker failed ({err})",
err=err)
# in any case, forward the error ..
worker.ready.errback(err)
d.addCallbacks(on_connect_success, on_connect_error)
return worker.ready
def start_router_component(self, id, config, details=None):
"""
Start an app component in this router worker.
:param id: The ID of the component to start.
:type id: str
:param config: The component configuration.
:type config: obj
"""
self.log.debug("{}.start_router_component".format(self.__class__.__name__),
id=id, config=config)
# prohibit starting a component twice
#
if id in self.components:
emsg = "Could not start component: a component with ID '{}'' is already running (or starting)".format(id)
self.log.error(emsg)
raise ApplicationError(u'crossbar.error.already_running', emsg)
# check configuration
#
try:
checkconfig.check_router_component(config)
except Exception as e:
emsg = "Invalid router component configuration: {}".format(e)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.invalid_configuration", emsg)
else:
self.log.debug("Starting {type}-component on router.",
type=config['type'])
# resolve references to other entities
#
references = {}
for ref in config.get('references', []):
ref_type, ref_id = ref.split(':')
if ref_type == u'connection':
if ref_id in self._connections:
references[ref] = self._connections[ref_id]
else:
emsg = "cannot resolve reference '{}' - no '{}' with ID '{}'".format(ref, ref_type, ref_id)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.invalid_configuration", emsg)
else:
emsg = "cannot resolve reference '{}' - invalid reference type '{}'".format(ref, ref_type)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.invalid_configuration", emsg)
# create component config
#
realm = config['realm']
extra = config.get('extra', None)
component_config = ComponentConfig(realm=realm, extra=extra)
create_component = _appsession_loader(config)
# .. and create and add an WAMP application session to
# run the component next to the router
#
try:
session = create_component(component_config)
# any exception spilling out from user code in onXXX handlers is fatal!
def panic(fail, msg):
self.log.error(
"Fatal error in component: {msg} - {log_failure.value}",
msg=msg, log_failure=fail
)
session.disconnect()
session._swallow_error = panic
except Exception:
self.log.error(
"Component instantiation failed",
log_failure=Failure(),
)
raise
# Note that 'join' is fired to listeners *before* onJoin runs,
# so if you do 'yield self.leave()' in onJoin we'll still
# publish "started" before "stopped".
def publish_stopped(session, stop_details):
self.log.info(
"stopped component: {session} id={session_id}",
session=class_name(session),
session_id=session._session_id,
)
topic = self._uri_prefix + '.container.on_component_stop'
event = {u'id': id}
caller = details.caller if details else None
self.publish(topic, event, options=PublishOptions(exclude=caller))
return event
def publish_started(session, start_details):
self.log.info(
"started component: {session} id={session_id}",
session=class_name(session),
session_id=session._session_id,
)
topic = self._uri_prefix + '.container.on_component_start'
event = {u'id': id}
caller = details.caller if details else None
self.publish(topic, event, options=PublishOptions(exclude=caller))
return event
session.on('leave', publish_stopped)
session.on('join', publish_started)
self.components[id] = RouterComponent(id, config, session)
self._router_session_factory.add(session, authrole=config.get('role', u'anonymous'))
self.log.debug(
"Added component {id} (type '{name}')",
id=id,
name=class_name(session),
)
def _start_native_worker(self, worker_type, worker_id, worker_options=None, details=None):
# prohibit starting a worker twice
#
if worker_id in self._workers:
emsg = "Could not start worker: a worker with ID '{}' is already running (or starting)".format(worker_id)
self.log.error(emsg)
raise ApplicationError(u'crossbar.error.worker_already_running', emsg)
# check worker options
#
options = worker_options or {}
try:
if worker_type in self._node._native_workers:
if self._node._native_workers[worker_type]['checkconfig_options']:
self._node._native_workers[worker_type]['checkconfig_options'](self.personality, options)
else:
raise Exception('No checkconfig_options for worker type "{worker_type}" implemented!'.format(worker_type=worker_type))
else:
raise Exception('invalid worker type "{}"'.format(worker_type))
except Exception as e:
emsg = "Could not start native worker: invalid configuration ({})".format(e)
self.log.error(emsg)
raise ApplicationError(u'crossbar.error.invalid_configuration', emsg)
# the fully qualified worker class as a string
worker_class = qual(self._node._native_workers[worker_type]['worker_class'])
# allow override Python executable from options
#
if 'python' in options:
exe = options['python']
# the executable must be an absolute path, e.g. /home/oberstet/pypy-2.2.1-linux64/bin/pypy
#
if not os.path.isabs(exe):
emsg = "Invalid worker configuration: python executable '{}' must be an absolute path".format(exe)
self.log.error(emsg)
raise ApplicationError(u'crossbar.error.invalid_configuration', emsg)
# of course the path must exist and actually be executable
#
if not (os.path.isfile(exe) and os.access(exe, os.X_OK)):
emsg = "Invalid worker configuration: python executable '{}' does not exist or isn't an executable".format(exe)
self.log.error(emsg)
raise ApplicationError(u'crossbar.error.invalid_configuration', emsg)
else:
exe = sys.executable
# allow override default Python module search paths from options
#
if 'pythonpath' in options:
pythonpaths_to_add = [os.path.abspath(os.path.join(self._node._cbdir, p)) for p in options.get('pythonpath', [])]
else:
pythonpaths_to_add = []
# assemble command line for forking the worker
#
# all native workers (routers and containers for now) start
# from the same script in crossbar/worker/process.py or
# from the command "crossbar _exec_worker" when crossbar is
# running from a frozen executable (single-file, pyinstaller, etc)
#
if getattr(sys, 'frozen', False):
# if we are inside a frozen crossbar executable, we need to invoke
# the crossbar executable with a command ("_exec_worker")
args = [exe, self._node.personality.NAME, "_exec_worker"]
else:
# we are invoking via "-m" so that .pyc files, __pycache__
# etc work properly. this works everywhere, but frozen executables
args = [exe, "-u", "-m", "crossbar.worker.main"]
args.extend(["--cbdir", self._node._cbdir])
args.extend(["--node", str(self._node._node_id)])
args.extend(["--worker", str(worker_id)])
args.extend(["--realm", self._realm])
args.extend(["--personality", class_name(self._node.personality)])
args.extend(["--klass", worker_class])
args.extend(["--loglevel", get_global_log_level()])
if self._node.options.debug_lifecycle:
args.append("--debug-lifecycle")
if self._node.options.debug_programflow:
args.append("--debug-programflow")
if "shutdown" in options:
args.extend(["--shutdown", options["shutdown"]])
# Node-level callback to inject worker arguments
#
self._node._extend_worker_args(args, options)
# allow override worker process title from options
#
if options.get('title', None):
args.extend(['--title', options['title']])
# forward explicit reactor selection
#
if 'reactor' in options and sys.platform in options['reactor']:
args.extend(['--reactor', options['reactor'][sys.platform]])
# FIXME
# elif self._node.options.reactor:
# args.extend(['--reactor', self._node.options.reactor])
# create worker process environment
#
worker_env = create_process_env(options)
# We need to use the same PYTHONPATH we were started with, so we can
# find the Crossbar we're working with -- it may not be the same as the
# one on the default path
worker_env["PYTHONPATH"] = os.pathsep.join(pythonpaths_to_add + sys.path)
# log name of worker
#
worker_logname = self._node._native_workers[worker_type]['logname']
# each worker is run under its own dedicated WAMP auth role
#
worker_auth_role = u'crossbar.worker.{}'.format(worker_id)
# topic URIs used (later)
#
starting_topic = self._node._native_workers[worker_type]['topics']['starting']
started_topic = self._node._native_workers[worker_type]['topics']['started']
# add worker tracking instance to the worker map ..
#
WORKER = self._node._native_workers[worker_type]['class']
worker = WORKER(self, worker_id, details.caller, keeplog=options.get('traceback', None))
self._workers[worker_id] = worker
# create a (custom) process endpoint.
#
if platform.isWindows():
childFDs = None # Use the default Twisted ones
else:
# The communication between controller and container workers is
# using WAMP running over 2 pipes.
# For controller->native-worker traffic this runs over FD 0 (`stdin`)
# and for the native-worker->controller traffic, this runs over FD 3.
#
# Note: We use FD 3, not FD 1 (`stdout`) or FD 2 (`stderr`) for
# container->controller traffic, so that components running in the
# container which happen to write to `stdout` or `stderr` do not
# interfere with the container-controller communication.
childFDs = {0: "w", 1: "r", 2: "r", 3: "r"}
ep = WorkerProcessEndpoint(
self._node._reactor, exe, args, env=worker_env, worker=worker,
childFDs=childFDs)
# ready handling
#
def on_ready_success(worker_id):
self.log.info('{worker_type} worker "{worker_id}" process {pid} started',
worker_type=worker_logname, worker_id=worker.id, pid=worker.pid)
self._node._reactor.addSystemEventTrigger(
'before', 'shutdown',
self._cleanup_worker, self._node._reactor, worker,
)
worker.on_worker_started()
started_info = {
u'id': worker.id,
u'status': worker.status,
u'started': utcstr(worker.started),
u'who': worker.who,
}
# FIXME: make start of stats printer dependent on log level ..
if False:
worker.log_stats(5.)
self.publish(started_topic, started_info, options=PublishOptions(exclude=details.caller))
return started_info
def on_ready_error(err):
del self._workers[worker.id]
emsg = 'Failed to start native worker: {}'.format(err.value)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.cannot_start", emsg, worker.getlog())
worker.ready.addCallbacks(on_ready_success, on_ready_error)
def on_exit_success(_):
self.log.info("Node worker {worker.id} ended successfully", worker=worker)
# clear worker log
worker.log_stats(0)
# remove the dedicated node router authrole we dynamically
# added for the worker
self._node._drop_worker_role(worker_auth_role)
# remove our metadata tracking for the worker
del self._workers[worker.id]
# indicate that the worker excited successfully
return True
def on_exit_error(err):
self.log.info("Node worker {worker.id} ended with error ({err})", worker=worker, err=err)
# clear worker log
worker.log_stats(0)
# remove the dedicated node router authrole we dynamically
# added for the worker
self._node._drop_worker_role(worker_auth_role)
# remove our metadata tracking for the worker
del self._workers[worker.id]
# indicate that the worker excited with error
return False
def check_for_shutdown(was_successful):
self.log.info('Checking for node shutdown: worker_exit_success={worker_exit_success}, shutdown_requested={shutdown_requested}, node_shutdown_triggers={node_shutdown_triggers}', worker_exit_success=was_successful, shutdown_requested=self._shutdown_requested, node_shutdown_triggers=self._node._node_shutdown_triggers)
shutdown = self._shutdown_requested
# automatically shutdown node whenever a worker ended (successfully, or with error)
#
if NODE_SHUTDOWN_ON_WORKER_EXIT in self._node._node_shutdown_triggers:
self.log.info("Node worker ended, and trigger '{trigger}' is active: will shutdown node ..", trigger=NODE_SHUTDOWN_ON_WORKER_EXIT)
term_print('CROSSBAR:NODE_SHUTDOWN_ON_WORKER_EXIT')
shutdown = True
# automatically shutdown node when worker ended with error
#
elif not was_successful and NODE_SHUTDOWN_ON_WORKER_EXIT_WITH_ERROR in self._node._node_shutdown_triggers:
self.log.info("Node worker ended with error, and trigger '{trigger}' is active: will shutdown node ..", trigger=NODE_SHUTDOWN_ON_WORKER_EXIT_WITH_ERROR)
term_print('CROSSBAR:NODE_SHUTDOWN_ON_WORKER_EXIT_WITH_ERROR')
shutdown = True
# automatically shutdown node when no more workers are left
#
elif len(self._workers) == 0 and NODE_SHUTDOWN_ON_LAST_WORKER_EXIT in self._node._node_shutdown_triggers:
self.log.info("No more node workers running, and trigger '{trigger}' is active: will shutdown node ..", trigger=NODE_SHUTDOWN_ON_LAST_WORKER_EXIT)
term_print('CROSSBAR:NODE_SHUTDOWN_ON_LAST_WORKER_EXIT')
shutdown = True
# initiate shutdown (but only if we are not already shutting down)
#
if shutdown:
self.shutdown()
else:
self.log.info('Node will continue to run!')
d_on_exit = worker.exit.addCallbacks(on_exit_success, on_exit_error)
d_on_exit.addBoth(check_for_shutdown)
# create a transport factory for talking WAMP to the native worker
#
transport_factory = create_native_worker_client_factory(self._node._router_session_factory, worker_auth_role, worker.ready, worker.exit)
transport_factory.noisy = False
self._workers[worker_id].factory = transport_factory
# now (immediately before actually forking) signal the starting of the worker
#
starting_info = {
u'id': worker_id,
u'status': worker.status,
u'created': utcstr(worker.created),
u'who': worker.who,
}
# the caller gets a progressive result ..
if details.progress:
details.progress(starting_info)
# .. while all others get an event
self.publish(starting_topic, starting_info, options=PublishOptions(exclude=details.caller))
# only the following line will actually exec a new worker process - everything before is just setup
# for this moment:
self.log.info('Starting new managed worker process for {worker_logname} worker "{worker_id}"',
worker_id=worker_id, worker_logname=worker_logname)
self.log.debug('Starting new managed worker process for {worker_logname} worker "{worker_id}" using {exe} with args {args}',
worker_id=worker_id, worker_logname=worker_logname, exe=exe, args=args)
d = ep.connect(transport_factory)
def on_connect_success(proto):
# this seems to be called immediately when the child process
# has been forked. even if it then immediately fails because
# e.g. the executable doesn't even exist. in other words,
# I'm not sure under what conditions the deferred will errback ..
self.log.debug('Native worker "{worker_id}" connected',
worker_id=worker_id)
worker.on_worker_connected(proto)
# dynamically add a dedicated authrole to the router
# for the worker we've just started
self._node._add_worker_role(worker_auth_role, options)
def on_connect_error(err):
# not sure when this errback is triggered at all ..
self.log.error("Interal error: connection to forked native worker failed ({err})", err=err)
# in any case, forward the error ..
worker.ready.errback(err)
d.addCallbacks(on_connect_success, on_connect_error)
return worker.ready
def start_router_component(self, id, config, details=None):
"""
Start an app component in this router worker.
:param id: The ID of the component to start.
:type id: str
:param config: The component configuration.
:type config: dict
:param details: Call details.
:type details: :class:`autobahn.wamp.types.CallDetails`
"""
self.log.debug("{name}.start_router_component", name=self.__class__.__name__)
# prohibit starting a component twice
#
if id in self.components:
emsg = "Could not start component: a component with ID '{}'' is already running (or starting)".format(id)
self.log.error(emsg)
raise ApplicationError(u'crossbar.error.already_running', emsg)
# check configuration
#
try:
self.personality.check_router_component(self.personality, config)
except Exception as e:
emsg = "Invalid router component configuration: {}".format(e)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.invalid_configuration", emsg)
else:
self.log.debug("Starting {type}-component on router.",
type=config['type'])
# resolve references to other entities
#
references = {}
for ref in config.get('references', []):
ref_type, ref_id = ref.split(':')
if ref_type == u'connection':
if ref_id in self._connections:
references[ref] = self._connections[ref_id]
else:
emsg = "cannot resolve reference '{}' - no '{}' with ID '{}'".format(ref, ref_type, ref_id)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.invalid_configuration", emsg)
else:
emsg = "cannot resolve reference '{}' - invalid reference type '{}'".format(ref, ref_type)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.invalid_configuration", emsg)
# create component config
#
realm = config.get('realm', None)
assert isinstance(realm, str)
extra = config.get('extra', {})
assert isinstance(extra, dict)
# forward crossbar node base directory
extra['cbdir'] = self.config.extra.cbdir
# allow access to controller session
controller = self if self.config.extra.expose_controller else None
# expose an object shared between components
shared = self.components_shared if self.config.extra.expose_shared else None
# this is the component configuration provided to the components ApplicationSession
component_config = ComponentConfig(realm=realm,
extra=extra,
keyring=None,
controller=controller,
shared=shared)
# define component ctor function
try:
create_component = _appsession_loader(config)
except ApplicationError as e:
# for convenience, also log failed component loading
self.log.error(u'component loading failed', log_failure=Failure())
if u'No module named' in str(e):
self.log.error(u' Python module search paths:')
for path in e.kwargs['pythonpath']:
self.log.error(u' {path}', path=path)
raise
# .. and create and add an WAMP application session to
# run the component next to the router
try:
session = create_component(component_config)
# any exception spilling out from user code in onXXX handlers is fatal!
def panic(fail, msg):
self.log.error(
"Fatal error in component: {msg} - {log_failure.value}",
msg=msg, log_failure=fail
)
session.disconnect()
session._swallow_error = panic
except Exception:
self.log.error(
"Component instantiation failed",
log_failure=Failure(),
)
raise
# Note that 'join' is fired to listeners *before* onJoin runs,
# so if you do 'yield self.leave()' in onJoin we'll still
# publish "started" before "stopped".
def publish_stopped(session, stop_details):
self.log.info(
"stopped component: {session} id={session_id}",
session=class_name(session),
session_id=session._session_id,
)
topic = self._uri_prefix + '.on_component_stop'
event = {u'id': id}
caller = details.caller if details else None
self.publish(topic, event, options=PublishOptions(exclude=caller))
return event
def publish_started(session, start_details):
self.log.info(
"started component: {session} id={session_id}",
session=class_name(session),
session_id=session._session_id,
)
topic = self._uri_prefix + '.on_component_start'
event = {u'id': id}
caller = details.caller if details else None
self.publish(topic, event, options=PublishOptions(exclude=caller))
return event
session.on('leave', publish_stopped)
session.on('join', publish_started)
self.components[id] = RouterComponent(id, config, session)
router = self._router_factory.get(realm)
self._router_session_factory.add(session, router, authrole=config.get('role', u'anonymous'))
self.log.debug(
"Added component {id} (type '{name}')",
id=id,
name=class_name(session),
)