def __init__(self, node): # base ctor NativeProcess.__init__(self, config=None, reactor=node._reactor, personality=node.personality) # associated node self._node = node self._realm = node._realm self.cbdir = self._node._cbdir self._uri_prefix = u'crossbar' self._started = None self._pid = os.getpid() # map of worker processes: worker_id -> NativeWorkerProcess self._workers = {} # shutdown of node is requested, and further requests to shutdown (or start) # are denied self._shutdown_requested = False # when shutting down, this flags marks if the shutdown is graceful and clean, # and expected (under the node configuration/settings) or if the shutdown is # under error or unnormal conditions. this flag controls the final exit # code returned by crossbar: 0 in case of "clean shutdown", and 1 otherwise self._shutdown_was_clean = None # node-wide system monitor running here in the node controller self._smonitor = SystemMonitor()
def __init__(self, node): # base ctor NativeProcess.__init__(self, config=None, reactor=node._reactor, personality=node.personality) # associated node self._node = node self._realm = node._realm self.cbdir = self._node._cbdir self._uri_prefix = u'crossbar' self._started = None self._pid = os.getpid() # map of worker processes: worker_id -> NativeWorkerProcess self._workers = {} # shutdown of node is requested, and further requests to shutdown (or start) # are denied self._shutdown_requested = False # when shutting down, this flags marks if the shutdown is graceful and clean, # and expected (under the node configuration/settings) or if the shutdown is # under error or unnormal conditions. this flag controls the final exit # code returned by crossbar: 0 in case of "clean shutdown", and 1 otherwise self._shutdown_was_clean = None # node-wide system monitor running here in the node controller self._smonitor = SystemMonitor()
def __init__(self, node): # call base ctor extra = namedtuple('Extra', ['node', 'worker'])(node._node_id, 'controller') config = ComponentConfig(realm=node._realm, extra=extra) NativeProcess.__init__(self, config=config, reactor=node._reactor, personality=node.personality) # associated node self._node = node # node directory self.cbdir = self._node._cbdir # overwrite URI prefix for controller (normally: "crossbar.worker.<worker_id>") self._uri_prefix = 'crossbar' self._started = None self._pid = os.getpid() # map of worker processes: worker_id -> NativeWorkerProcess self._workers = { # add worker tracking instance to the worker map for the controller itself (!) .. # 'controller': self } # shutdown of node is requested, and further requests to shutdown (or start) # are denied self._shutdown_requested = False # when shutting down, this flags marks if the shutdown is graceful and clean, # and expected (under the node configuration/settings) or if the shutdown is # under error or unnormal conditions. this flag controls the final exit # code returned by crossbar: 0 in case of "clean shutdown", and 1 otherwise self._shutdown_was_clean = None # node-wide system monitor running here in the node controller self._smonitor = SystemMonitor()
class NodeController(NativeProcess): log = make_logger() WORKER_TYPE = u'controller' def __init__(self, node): # base ctor NativeProcess.__init__(self, config=None, reactor=node._reactor, personality=node.personality) # associated node self._node = node self._realm = node._realm self.cbdir = self._node._cbdir self._uri_prefix = u'crossbar' self._started = None self._pid = os.getpid() # map of worker processes: worker_id -> NativeWorkerProcess self._workers = {} # shutdown of node is requested, and further requests to shutdown (or start) # are denied self._shutdown_requested = False # when shutting down, this flags marks if the shutdown is graceful and clean, # and expected (under the node configuration/settings) or if the shutdown is # under error or unnormal conditions. this flag controls the final exit # code returned by crossbar: 0 in case of "clean shutdown", and 1 otherwise self._shutdown_was_clean = None # node-wide system monitor running here in the node controller self._smonitor = SystemMonitor() def onConnect(self): self.log.debug("Connected to node management router") NativeProcess.onConnect(self, False) # self.join(self.config.realm) self.join(self._realm) @inlineCallbacks def onJoin(self, details): from autobahn.wamp.types import SubscribeOptions self.log.debug("Joined realm '{realm}' on node management router", realm=details.realm) # When a (native) worker process has connected back to the router of # the node controller, the worker will publish this event # to signal it's readyness. # def on_worker_ready(res): worker_id = res['id'] if worker_id in self._workers: ready = self._workers[worker_id].ready if not ready.called: # fire the Deferred previously stored for # signaling "worker ready" ready.callback(worker_id) else: self.log.error( "Internal error: on_worker_ready() fired for process {process}, but already called earlier", process=worker_id) else: self.log.error( "Internal error: on_worker_ready() fired for process {process}, but no process with that ID", process=worker_id) self.subscribe(on_worker_ready, u'crossbar.worker..on_worker_ready', SubscribeOptions(match=u'wildcard')) yield NativeProcess.onJoin(self, details) # above upcall registers procedures we have marked with @wamp.register(None) # we need to catch SIGINT here to properly shutdown the # node explicitly (a Twisted system trigger wouldn't allow us to distinguish # different reasons/origins of exiting ..) def signal_handler(_signal, frame): if _signal == signal.SIGINT: # CTRL-C'ing Crossbar.io is considered "willful", and hence we want to exit cleanly self._shutdown_was_clean = True elif _signal == signal.SIGTERM: self._shutdown_was_clean = False else: # FIXME: can we run into others here? self._shutdown_was_clean = False self.log.warn( 'Controller received SIGINT [signal={signal}]: shutting down node [shutdown_was_clean={shutdown_was_clean}] ..', signal=_signal, shutdown_was_clean=self._shutdown_was_clean) # the following will shutdown the Twisted reactor in the end self.shutdown() signal.signal(signal.SIGINT, signal_handler) self.log.info('Signal handler installed on process {pid} thread {tid}', pid=os.getpid(), tid=threading.get_ident()) self._started = utcnow() self.publish(u"crossbar.on_ready") self.log.debug("Node controller ready") @wamp.register(None) def get_status(self, details=None): """ Return basic information about this node. :param details: Call details. :type details: :class:`autobahn.wamp.types.CallDetails` :returns: Information on the Crossbar.io node. :rtype: dict """ workers_by_type = {} for worker in self._workers.values(): if worker.TYPE not in workers_by_type: workers_by_type[worker.TYPE] = 0 workers_by_type[worker.TYPE] += 1 return { 'title': u'{} {}'.format(self.personality.TITLE, crossbar.__version__), 'started': self._started, 'controller_pid': self._pid, 'running_workers': len(self._workers), 'workers_by_type': workers_by_type, 'directory': self.cbdir, 'pubkey': self._node._node_key.public_key(), } @wamp.register(None) def get_system_stats(self, details=None): """ Return system statistics on this node. :param details: Call details. :type details: :class:`autobahn.wamp.types.CallDetails` :return: Current system statistics for this node. :rtype: dict """ self.log.debug("{cls}.get_stats", cls=self.__class__.__name__) return self._smonitor.poll() @wamp.register(None) @inlineCallbacks def shutdown(self, restart=False, mode=None, details=None): """ Explicitly stop this node. """ if self._shutdown_requested: # we're already shutting down .. ignore .. return self._shutdown_requested = True self.log.info( 'Node shutdown requested (restart={}, mode={}, reactor.running={}) ..' .format(restart, mode, self._reactor.running)) term_print('CROSSBAR:NODE_SHUTDOWN_REQUESTED') try: # shutdown any specific to the node controller yield self._shutdown(restart, mode) # node shutdown information shutdown_info = { u'node_id': self._node._node_id, u'restart': restart, u'mode': mode, u'who': details.caller if details else None, u'when': utcnow(), u'was_clean': self._shutdown_was_clean, } if self._node._shutdown_complete: self._node._shutdown_complete.callback(shutdown_info) # publish management API event yield self.publish(u'{}.on_shutdown'.format(self._uri_prefix), shutdown_info, options=PublishOptions( exclude=details.caller if details else None, acknowledge=True)) def stop_reactor(): try: self._reactor.stop() except ReactorNotRunning: pass _SHUTDOWN_DELAY = 0.2 self._reactor.callLater(_SHUTDOWN_DELAY, stop_reactor) except: self.log.failure() self._shutdown_requested = False raise else: returnValue(shutdown_info) # to be overridden in derived node classes to shutdown any # specifics (eg clients like for etcd or docker) def _shutdown(self, restart=False, mode=None): pass @wamp.register(None) def get_workers(self, filter_types=[], details=None): """ Returns the list of workers currently running on this node. :param filter_types: :returns: List of worker processes. :rtype: list[dict] """ assert filter_types is None or (type(filter_types) == list and type(ft) == str for ft in filter_types) if filter_types: ft = set(filter_types) worker_ids = [ worker_id for worker_id in self._workers if self._workers[worker_id].TYPE in ft ] else: worker_ids = self._workers.keys() return sorted(worker_ids) @wamp.register(None) def get_worker(self, worker_id, include_stats=False, details=None): """ Return detailed information about worker. :param worker_id: ID of worker to get information for. :type worker_id: str :param include_stats: If true, include worker run-time statistics. :type include_stats: bool """ if worker_id not in self._workers: emsg = "No worker with ID '{}'".format(worker_id) raise ApplicationError(u'crossbar.error.no_such_worker', emsg) now = datetime.utcnow() worker = self._workers[worker_id] worker_info = { u'id': worker.id, u'pid': worker.pid, u'type': worker.TYPE, u'status': worker.status, u'created': utcstr(worker.created), u'started': utcstr(worker.started), u'startup_time': (worker.started - worker.created).total_seconds() if worker.started else None, u'uptime': (now - worker.started).total_seconds() if worker.started else None, } if include_stats: stats = {u'controller_traffic': worker.get_stats()} worker_info[u'stats'] = stats return worker_info @wamp.register(None) def start_worker(self, worker_id, worker_type, worker_options=None, details=None): """ Start a new worker process in the node. """ self.log.info( 'Starting {worker_type} worker {worker_id} {worker_klass}', worker_type=worker_type, worker_id=hlid(worker_id), worker_klass=hltype(NodeController.start_worker)) if type(worker_id) != str or worker_id in [u'controller', u'']: raise Exception('invalid worker ID "{}"'.format(worker_id)) if worker_type == u'guest': return self._start_guest_worker(worker_id, worker_options, details=details) elif worker_type in self._node._native_workers: return self._start_native_worker(worker_type, worker_id, worker_options, details=details) else: raise Exception('invalid worker type "{}"'.format(worker_type)) @wamp.register(None) def stop_worker(self, worker_id, kill=False, details=None): """ Stop a running worker. :param worker_id: ID of worker to stop. :type worker_id: str :param kill: If ``True``, kill the process. Otherwise, gracefully shut down the worker. :type kill: bool :returns: Stopping information from the worker. :rtype: dict """ if worker_id not in self._workers: emsg = "No worker with ID '{}'".format(worker_id) raise ApplicationError(u'crossbar.error.no_such_worker', emsg) worker = self._workers[worker_id] if worker.TYPE in self._node._native_workers: return self._stop_native_worker(worker_id, kill=kill, details=details) elif worker.TYPE == u'guest': return self._stop_guest_worker(worker_id, kill=kill, details=details) else: # should not arrive here raise Exception('logic error') @wamp.register(None) def get_worker_log(self, worker_id, limit=100, details=None): """ Get buffered log for a worker. :param worker_id: The worker ID to get log output for. :type worker_id: str :param limit: Limit the amount of log entries returned to the last N entries. :type limit: int :returns: Buffered log for worker. :rtype: list """ if worker_id not in self._workers: emsg = "No worker with ID '{}'".format(worker_id) raise ApplicationError(u'crossbar.error.no_such_worker', emsg) return self._workers[worker_id].getlog(limit) def _start_native_worker(self, worker_type, worker_id, worker_options=None, details=None): # prohibit starting a worker twice # if worker_id in self._workers: emsg = "Could not start worker: a worker with ID '{}' is already running (or starting)".format( worker_id) self.log.error(emsg) raise ApplicationError(u'crossbar.error.worker_already_running', emsg) # check worker options # options = worker_options or {} try: if worker_type in self._node._native_workers: if self._node._native_workers[worker_type][ 'checkconfig_options']: self._node._native_workers[worker_type][ 'checkconfig_options'](self.personality, options) else: raise Exception( 'No checkconfig_options for worker type "{worker_type}" implemented!' .format(worker_type=worker_type)) else: raise Exception('invalid worker type "{}"'.format(worker_type)) except Exception as e: emsg = "Could not start native worker: invalid configuration ({})".format( e) self.log.error(emsg) raise ApplicationError(u'crossbar.error.invalid_configuration', emsg) # the fully qualified worker class as a string worker_class = qual( self._node._native_workers[worker_type]['worker_class']) # allow override Python executable from options # if 'python' in options: exe = options['python'] # the executable must be an absolute path, e.g. /home/oberstet/pypy-2.2.1-linux64/bin/pypy # if not os.path.isabs(exe): emsg = "Invalid worker configuration: python executable '{}' must be an absolute path".format( exe) self.log.error(emsg) raise ApplicationError(u'crossbar.error.invalid_configuration', emsg) # of course the path must exist and actually be executable # if not (os.path.isfile(exe) and os.access(exe, os.X_OK)): emsg = "Invalid worker configuration: python executable '{}' does not exist or isn't an executable".format( exe) self.log.error(emsg) raise ApplicationError(u'crossbar.error.invalid_configuration', emsg) else: exe = sys.executable # allow override default Python module search paths from options # if 'pythonpath' in options: pythonpaths_to_add = [ os.path.abspath(os.path.join(self._node._cbdir, p)) for p in options.get('pythonpath', []) ] else: pythonpaths_to_add = [] # assemble command line for forking the worker # # all native workers (routers and containers for now) start # from the same script in crossbar/worker/process.py or # from the command "crossbar _exec_worker" when crossbar is # running from a frozen executable (single-file, pyinstaller, etc) # if getattr(sys, 'frozen', False): # if we are inside a frozen crossbar executable, we need to invoke # the crossbar executable with a command ("_exec_worker") args = [exe, self._node.personality.NAME, "_exec_worker"] else: # we are invoking via "-m" so that .pyc files, __pycache__ # etc work properly. this works everywhere, but frozen executables args = [exe, "-u", "-m", "crossbar.worker.main"] args.extend(["--cbdir", self._node._cbdir]) args.extend(["--node", str(self._node._node_id)]) args.extend(["--worker", str(worker_id)]) args.extend(["--realm", self._realm]) args.extend(["--personality", class_name(self._node.personality)]) args.extend(["--klass", worker_class]) args.extend(["--loglevel", get_global_log_level()]) if self._node.options.debug_lifecycle: args.append("--debug-lifecycle") if self._node.options.debug_programflow: args.append("--debug-programflow") if self._node.options.enable_vmprof: args.append("--vmprof") if "shutdown" in options: args.extend(["--shutdown", options["shutdown"]]) # Node-level callback to inject worker arguments # self._node._extend_worker_args(args, options) # allow override worker process title from options # if options.get('title', None): args.extend(['--title', options['title']]) # forward explicit reactor selection # if 'reactor' in options and sys.platform in options['reactor']: args.extend(['--reactor', options['reactor'][sys.platform]]) # FIXME # elif self._node.options.reactor: # args.extend(['--reactor', self._node.options.reactor]) # create worker process environment # worker_env = create_process_env(options) # We need to use the same PYTHONPATH we were started with, so we can # find the Crossbar we're working with -- it may not be the same as the # one on the default path worker_env["PYTHONPATH"] = os.pathsep.join(pythonpaths_to_add + sys.path) # log name of worker # worker_logname = self._node._native_workers[worker_type]['logname'] # each worker is run under its own dedicated WAMP auth role # worker_auth_role = u'crossbar.worker.{}'.format(worker_id) # topic URIs used (later) # starting_topic = self._node._native_workers[worker_type]['topics'][ 'starting'] started_topic = self._node._native_workers[worker_type]['topics'][ 'started'] # add worker tracking instance to the worker map .. # WORKER = self._node._native_workers[worker_type]['class'] worker = WORKER(self, worker_id, details.caller, keeplog=options.get('traceback', None)) self._workers[worker_id] = worker # create a (custom) process endpoint. # if platform.isWindows(): childFDs = None # Use the default Twisted ones else: # The communication between controller and container workers is # using WAMP running over 2 pipes. # For controller->native-worker traffic this runs over FD 0 (`stdin`) # and for the native-worker->controller traffic, this runs over FD 3. # # Note: We use FD 3, not FD 1 (`stdout`) or FD 2 (`stderr`) for # container->controller traffic, so that components running in the # container which happen to write to `stdout` or `stderr` do not # interfere with the container-controller communication. childFDs = {0: "w", 1: "r", 2: "r", 3: "r"} ep = WorkerProcessEndpoint(self._node._reactor, exe, args, env=worker_env, worker=worker, childFDs=childFDs) # ready handling # def on_ready_success(worker_id): self.log.debug( '{worker_type} worker "{worker_id}" process {pid} started', worker_type=worker_logname, worker_id=worker.id, pid=worker.pid) self._node._reactor.addSystemEventTrigger( 'before', 'shutdown', self._cleanup_worker, self._node._reactor, worker, ) worker.on_worker_started() started_info = { u'id': worker.id, u'status': worker.status, u'started': utcstr(worker.started), u'who': worker.who, u'pid': worker.pid, u'startup_time': (worker.started - worker.created).total_seconds() if worker.started else None } # FIXME: make start of stats printer dependent on log level .. if False: worker.log_stats(5.) self.publish(started_topic, started_info, options=PublishOptions(exclude=details.caller)) return started_info def on_ready_error(err): del self._workers[worker.id] emsg = 'Failed to start native worker: {}'.format(err.value) self.log.error(emsg) raise ApplicationError(u"crossbar.error.cannot_start", emsg, worker.getlog()) worker.ready.addCallbacks(on_ready_success, on_ready_error) def on_exit_success(_): self.log.info("Node worker {worker.id} ended successfully", worker=worker) # clear worker log worker.log_stats(0) # remove the dedicated node router authrole we dynamically # added for the worker self._node._drop_worker_role(worker_auth_role) # remove our metadata tracking for the worker del self._workers[worker.id] # indicate that the worker excited successfully return True def on_exit_error(err): self.log.info("Node worker {worker.id} ended with error ({err})", worker=worker, err=err) # clear worker log worker.log_stats(0) # remove the dedicated node router authrole we dynamically # added for the worker self._node._drop_worker_role(worker_auth_role) # remove our metadata tracking for the worker del self._workers[worker.id] # indicate that the worker excited with error return False def check_for_shutdown(was_successful): self.log.info( 'Checking for node shutdown: worker_exit_success={worker_exit_success}, shutdown_requested={shutdown_requested}, node_shutdown_triggers={node_shutdown_triggers}', worker_exit_success=was_successful, shutdown_requested=self._shutdown_requested, node_shutdown_triggers=self._node._node_shutdown_triggers) shutdown = self._shutdown_requested # automatically shutdown node whenever a worker ended (successfully, or with error) # if NODE_SHUTDOWN_ON_WORKER_EXIT in self._node._node_shutdown_triggers: self.log.info( "Node worker ended, and trigger '{trigger}' is active: will shutdown node ..", trigger=NODE_SHUTDOWN_ON_WORKER_EXIT) term_print('CROSSBAR:NODE_SHUTDOWN_ON_WORKER_EXIT') shutdown = True # automatically shutdown node when worker ended with error # elif not was_successful and NODE_SHUTDOWN_ON_WORKER_EXIT_WITH_ERROR in self._node._node_shutdown_triggers: self.log.info( "Node worker ended with error, and trigger '{trigger}' is active: will shutdown node ..", trigger=NODE_SHUTDOWN_ON_WORKER_EXIT_WITH_ERROR) term_print('CROSSBAR:NODE_SHUTDOWN_ON_WORKER_EXIT_WITH_ERROR') shutdown = True # automatically shutdown node when no more workers are left # elif len( self._workers ) == 0 and NODE_SHUTDOWN_ON_LAST_WORKER_EXIT in self._node._node_shutdown_triggers: self.log.info( "No more node workers running, and trigger '{trigger}' is active: will shutdown node ..", trigger=NODE_SHUTDOWN_ON_LAST_WORKER_EXIT) term_print('CROSSBAR:NODE_SHUTDOWN_ON_LAST_WORKER_EXIT') shutdown = True # initiate shutdown (but only if we are not already shutting down) # if shutdown: self.shutdown() else: self.log.info('Node will continue to run!') d_on_exit = worker.exit.addCallbacks(on_exit_success, on_exit_error) d_on_exit.addBoth(check_for_shutdown) # create a transport factory for talking WAMP to the native worker # transport_factory = create_native_worker_client_factory( self._node._router_session_factory, worker_auth_role, worker.ready, worker.exit) transport_factory.noisy = False self._workers[worker_id].factory = transport_factory # now (immediately before actually forking) signal the starting of the worker # starting_info = { u'id': worker_id, u'status': worker.status, u'created': utcstr(worker.created), u'who': worker.who, } # the caller gets a progressive result .. if details.progress: details.progress(starting_info) # .. while all others get an event self.publish(starting_topic, starting_info, options=PublishOptions(exclude=details.caller)) # only the following line will actually exec a new worker process - everything before is just setup # for this moment: self.log.debug( 'Starting new managed worker process for {worker_logname} worker "{worker_id}" using {exe} with args {args}', worker_id=worker_id, worker_logname=worker_logname, exe=exe, args=args) d = ep.connect(transport_factory) def on_connect_success(proto): # this seems to be called immediately when the child process # has been forked. even if it then immediately fails because # e.g. the executable doesn't even exist. in other words, # I'm not sure under what conditions the deferred will errback .. self.log.debug('Native worker "{worker_id}" connected', worker_id=worker_id) worker.on_worker_connected(proto) # dynamically add a dedicated authrole to the router # for the worker we've just started self._node._add_worker_role(worker_auth_role, options) def on_connect_error(err): # not sure when this errback is triggered at all .. self.log.error( "Internal error: connection to forked native worker failed ({err})", err=err) # in any case, forward the error .. worker.ready.errback(err) d.addCallbacks(on_connect_success, on_connect_error) return worker.ready @staticmethod def _cleanup_worker(reactor, worker): """ This is called during reactor shutdown and ensures we wait for our subprocesses to shut down nicely. """ log = make_logger() try: log.info("sending TERM to subprocess {pid}", pid=worker.pid) worker.proto.transport.signalProcess('TERM') # wait for the subprocess to shutdown; could add a timeout # after which we send a KILL maybe? d = Deferred() def protocol_closed(_): log.debug("{pid} exited", pid=worker.pid) d.callback(None) # await worker's timely demise worker.exit.addCallback(protocol_closed) def timeout(tried): if d.called: return log.info("waiting for {pid} to exit...", pid=worker.pid) reactor.callLater(1, timeout, tried + 1) if tried > 20: # or just wait forever? log.info("Sending SIGKILL to {pid}", pid=worker.pid) try: worker.proto.transport.signalProcess('KILL') except ProcessExitedAlready: pass # ignore; it's already dead d.callback(None) # or recurse more? timeout(0) return d except ProcessExitedAlready: pass # ignore; it's already dead @inlineCallbacks def _stop_native_worker(self, worker_id, kill, details=None): if worker_id not in self._workers or not isinstance( self._workers[worker_id], NativeWorkerProcess): emsg = "Could not stop native worker: no native worker with ID '{}' currently running".format( worker_id) raise ApplicationError(u'crossbar.error.worker_not_running', emsg) worker = self._workers[worker_id] if worker.status != 'started': emsg = "Could not stop native worker: worker with ID '{}' is not in status 'started', but status: '{}')".format( worker_id, worker.status) raise ApplicationError(u'crossbar.error.worker_not_running', emsg) stop_info = { u'id': worker.id, u'type': worker.TYPE, u'kill': kill, u'who': details.caller if details else None, u'when': utcnow(), } # publish management API event # yield self.publish(u'{}.on_stop_requested'.format(self._uri_prefix), stop_info, options=PublishOptions( exclude=details.caller if details else None, acknowledge=True)) # send SIGKILL or SIGTERM to worker # if kill: self.log.info("Killing {worker_type} worker with ID '{worker_id}'", worker_type=worker.TYPE, worker_id=worker_id) self._workers[worker_id].proto.transport.signalProcess("KILL") else: self.log.info( "Stopping {worker_type} worker with ID '{worker_id}'", worker_type=worker.TYPE, worker_id=worker_id) self._workers[worker_id].factory.stopFactory() self._workers[worker_id].proto.transport.signalProcess('TERM') # wait until the worker is actually done before we return from # this call yield self._workers[worker_id].proto.is_closed returnValue(stop_info) def _start_guest_worker(self, worker_id, worker_config, details=None): """ Start a new guest process on this node. :param config: The guest process configuration. :type config: dict :returns: The PID of the new process. """ # prohibit starting a worker twice # if worker_id in self._workers: emsg = "Could not start worker: a worker with ID '{}' is already running (or starting)".format( worker_id) self.log.error(emsg) raise ApplicationError(u'crossbar.error.worker_already_running', emsg) try: self.personality.check_guest(self.personality, worker_config) except Exception as e: raise ApplicationError( u'crossbar.error.invalid_configuration', 'invalid guest worker configuration: {}'.format(e)) options = worker_config.get('options', {}) # guest process working directory # workdir = self._node._cbdir if 'workdir' in options: workdir = os.path.join(workdir, options['workdir']) workdir = os.path.abspath(workdir) # guest process executable and command line arguments # # first try to configure the fully qualified path for the guest # executable by joining workdir and configured exectuable .. exe = os.path.abspath( os.path.join(workdir, worker_config['executable'])) if check_executable(exe): self.log.info( "Using guest worker executable '{exe}' (executable path taken from configuration)", exe=exe) else: # try to detect the fully qualified path for the guest # executable by doing a "which" on the configured executable name exe = which(worker_config['executable']) if exe is not None and check_executable(exe): self.log.info( "Using guest worker executable '{exe}' (executable path detected from environment)", exe=exe) else: emsg = "Could not start worker: could not find and executable for '{}'".format( worker_config['executable']) self.log.error(emsg) raise ApplicationError(u'crossbar.error.invalid_configuration', emsg) # guest process command line arguments # args = [exe] args.extend(worker_config.get('arguments', [])) # guest process environment # worker_env = create_process_env(options) # log name of worker # worker_logname = 'Guest' # topic URIs used (later) # starting_topic = u'{}.on_guest_starting'.format(self._uri_prefix) started_topic = u'{}.on_guest_started'.format(self._uri_prefix) # add worker tracking instance to the worker map .. # worker = GuestWorkerProcess(self, worker_id, details.caller, keeplog=options.get('traceback', None)) self._workers[worker_id] = worker # create a (custom) process endpoint # ep = WorkerProcessEndpoint(self._node._reactor, exe, args, path=workdir, env=worker_env, worker=worker) # ready handling # def on_ready_success(proto): self.log.info('{worker_logname} worker "{worker_id}" started', worker_logname=worker_logname, worker_id=worker.id) worker.on_worker_started(proto) self._node._reactor.addSystemEventTrigger( 'before', 'shutdown', self._cleanup_worker, self._node._reactor, worker, ) # directory watcher # if 'watch' in options: if HAS_FS_WATCHER: # assemble list of watched directories watched_dirs = [] for d in options['watch'].get('directories', []): watched_dirs.append( os.path.abspath(os.path.join(self._node._cbdir, d))) worker.watch_timeout = options['watch'].get('timeout', 1) # create a filesystem watcher worker.watcher = FilesystemWatcher( workdir, watched_dirs=watched_dirs) # make sure to stop the watch upon Twisted being shut down def on_shutdown(): worker.watcher.stop() self._node._reactor.addSystemEventTrigger( 'before', 'shutdown', on_shutdown) # this handler will get fired by the watcher upon detecting an FS event def on_filesystem_change(fs_event): worker.watcher.stop() proto.signal('TERM') if options['watch'].get('action', None) == 'restart': self.log.info( "Filesystem watcher detected change {fs_event} - restarting guest in {watch_timeout} seconds ..", fs_event=fs_event, watch_timeout=worker.watch_timeout) # Add a timeout large enough (perhaps add a config option later) self._node._reactor.callLater( worker.watch_timeout, self.start_worker, worker_id, worker_config, details) # Shut the worker down, after the restart event is scheduled # FIXME: all workers should have a stop() method .. # -> 'GuestWorkerProcess' object has no attribute 'stop' # worker.stop() else: self.log.info( "Filesystem watcher detected change {fs_event} - no action taken!", fs_event=fs_event) # now start watching .. worker.watcher.start(on_filesystem_change) else: self.log.warn( "Cannot watch directories for changes - feature not available" ) # assemble guest worker startup information # started_info = { u'id': worker.id, u'status': worker.status, u'started': utcstr(worker.started), u'who': worker.who, } self.publish(started_topic, started_info, options=PublishOptions(exclude=details.caller)) return started_info def on_ready_error(err): del self._workers[worker.id] emsg = 'Failed to start guest worker: {}'.format(err.value) self.log.error(emsg) raise ApplicationError(u"crossbar.error.cannot_start", emsg, ep.getlog()) worker.ready.addCallbacks(on_ready_success, on_ready_error) def on_exit_success(res): self.log.info("Guest {worker_id} exited with success", worker_id=worker.id) del self._workers[worker.id] def on_exit_error(err): self.log.error("Guest {worker_id} exited with error {err.value}", worker_id=worker.id, err=err) del self._workers[worker.id] worker.exit.addCallbacks(on_exit_success, on_exit_error) # create a transport factory for talking WAMP to the native worker # transport_factory = create_guest_worker_client_factory( worker_config, worker.ready, worker.exit) transport_factory.noisy = False self._workers[worker_id].factory = transport_factory # now (immediately before actually forking) signal the starting of the worker # starting_info = { u'id': worker_id, u'status': worker.status, u'created': utcstr(worker.created), u'who': worker.who, } # the caller gets a progressive result .. if details.progress: details.progress(starting_info) # .. while all others get an event self.publish(starting_topic, starting_info, options=PublishOptions(exclude=details.caller)) # now actually fork the worker .. # self.log.info('{worker_logname} "{worker_id}" process starting ..', worker_logname=worker_logname, worker_id=worker_id) self.log.debug( '{worker_logname} "{worker_id}" process using command line "{cli}" ..', worker_logname=worker_logname, worker_id=worker_id, cli=' '.join(args)) d = ep.connect(transport_factory) def on_connect_success(proto): # this seems to be called immediately when the child process # has been forked. even if it then immediately fails because # e.g. the executable doesn't even exist. in other words, # I'm not sure under what conditions the deferred will # errback - probably only if the forking of a new process fails # at OS level due to out of memory conditions or such. self.log.debug('{worker_logname} "{worker_id}" connected', worker_logname=worker_logname, worker_id=worker_id) # do not comment this: it will lead to on_worker_started being called # _before_ on_worker_connected, and we don't need it! # worker.on_worker_connected(proto) def on_connect_error(err): # not sure when this errback is triggered at all .. see above. self.log.failure( "Internal error: connection to forked guest worker failed ({log_failure.value})", ) # in any case, forward the error .. worker.ready.errback(err) d.addCallbacks(on_connect_success, on_connect_error) return worker.ready def _stop_guest_worker(self, worker_id, kill=False, details=None): """ Stops a currently running guest worker. :param worker_id: The ID of the guest worker to stop. :type worker_id: str """ self.log.debug("stop_guest({worker_id}, kill={kill})", worker_id=worker_id, kill=kill) if worker_id not in self._workers or self._workers[ worker_id].TYPE != 'guest': emsg = "Could not stop guest worker: no guest worker with ID '{}' currently running".format( worker_id) raise ApplicationError(u'crossbar.error.worker_not_running', emsg) worker = self._workers[worker_id] stop_info = { u'id': worker.id, u'type': u'guest', u'kill': kill, u'who': details.caller if details else None, u'when': utcnow(), } try: if kill: self._workers[worker_id].proto.transport.signalProcess("KILL") else: self._workers[worker_id].proto.transport.loseConnection() self._workers[worker_id].proto.transport.signalProcess("TERM") except Exception as e: emsg = "Could not stop guest worker with ID '{}': {}".format( worker_id, e) raise ApplicationError(u'crossbar.error.stop_worker_failed', emsg) else: del self._workers[worker_id] return stop_info
class NodeController(NativeProcess): log = make_logger() WORKER_TYPE = u'controller' def __init__(self, node): # base ctor NativeProcess.__init__(self, config=None, reactor=node._reactor, personality=node.personality) # associated node self._node = node self._realm = node._realm self.cbdir = self._node._cbdir self._uri_prefix = u'crossbar' self._started = None self._pid = os.getpid() # map of worker processes: worker_id -> NativeWorkerProcess self._workers = {} # shutdown of node is requested, and further requests to shutdown (or start) # are denied self._shutdown_requested = False # when shutting down, this flags marks if the shutdown is graceful and clean, # and expected (under the node configuration/settings) or if the shutdown is # under error or unnormal conditions. this flag controls the final exit # code returned by crossbar: 0 in case of "clean shutdown", and 1 otherwise self._shutdown_was_clean = None # node-wide system monitor running here in the node controller self._smonitor = SystemMonitor() def onConnect(self): self.log.debug("Connected to node management router") NativeProcess.onConnect(self, False) # self.join(self.config.realm) self.join(self._realm) @inlineCallbacks def onJoin(self, details): from autobahn.wamp.types import SubscribeOptions self.log.debug("Joined realm '{realm}' on node management router", realm=details.realm) # When a (native) worker process has connected back to the router of # the node controller, the worker will publish this event # to signal it's readyness. # def on_worker_ready(res): worker_id = res['id'] if worker_id in self._workers: ready = self._workers[worker_id].ready if not ready.called: # fire the Deferred previously stored for # signaling "worker ready" ready.callback(worker_id) else: self.log.error("Internal error: on_worker_ready() fired for process {process}, but already called earlier", process=worker_id) else: self.log.error("Internal error: on_worker_ready() fired for process {process}, but no process with that ID", process=worker_id) self.subscribe(on_worker_ready, u'crossbar.worker..on_worker_ready', SubscribeOptions(match=u'wildcard')) yield NativeProcess.onJoin(self, details) # above upcall registers procedures we have marked with @wamp.register(None) # we need to catch SIGINT here to properly shutdown the # node explicitly (a Twisted system trigger wouldn't allow us to distinguish # different reasons/origins of exiting ..) def signal_handler(_signal, frame): if _signal == signal.SIGINT: # CTRL-C'ing Crossbar.io is considered "willful", and hence we want to exit cleanly self._shutdown_was_clean = True elif _signal == signal.SIGTERM: self._shutdown_was_clean = False else: # FIXME: can we run into others here? self._shutdown_was_clean = False self.log.warn('Controller received SIGINT [signal={signal}]: shutting down node [shutdown_was_clean={shutdown_was_clean}] ..', signal=_signal, shutdown_was_clean=self._shutdown_was_clean) # the following will shutdown the Twisted reactor in the end self.shutdown() signal.signal(signal.SIGINT, signal_handler) self.log.info('Signal handler installed on process {pid} thread {tid}', pid=os.getpid(), tid=threading.get_ident()) self._started = utcnow() self.publish(u"crossbar.on_ready") self.log.debug("Node controller ready") @wamp.register(None) def get_status(self, details=None): """ Return basic information about this node. :param details: Call details. :type details: :class:`autobahn.wamp.types.CallDetails` :returns: Information on the Crossbar.io node. :rtype: dict """ workers_by_type = {} for worker in self._workers.values(): if worker.TYPE not in workers_by_type: workers_by_type[worker.TYPE] = 0 workers_by_type[worker.TYPE] += 1 return { 'title': u'{} {}'.format(self.personality.TITLE, crossbar.__version__), 'started': self._started, 'controller_pid': self._pid, 'running_workers': len(self._workers), 'workers_by_type': workers_by_type, 'directory': self.cbdir, 'pubkey': self._node._node_key.public_key(), } @wamp.register(None) def get_system_stats(self, details=None): """ Return system statistics on this node. :param details: Call details. :type details: :class:`autobahn.wamp.types.CallDetails` :return: Current system statistics for this node. :rtype: dict """ self.log.debug("{cls}.get_stats", cls=self.__class__.__name__) return self._smonitor.poll() @wamp.register(None) @inlineCallbacks def shutdown(self, restart=False, mode=None, details=None): """ Explicitly stop this node. """ if self._shutdown_requested: # we're already shutting down .. ignore .. return self._shutdown_requested = True self.log.info('Node shutdown requested (restart={}, mode={}, reactor.running={}) ..'.format( restart, mode, self._reactor.running)) term_print('CROSSBAR:NODE_SHUTDOWN_REQUESTED') try: # shutdown any specific to the node controller yield self._shutdown(restart, mode) # node shutdown information shutdown_info = { u'node_id': self._node._node_id, u'restart': restart, u'mode': mode, u'who': details.caller if details else None, u'when': utcnow(), u'was_clean': self._shutdown_was_clean, } if self._node._shutdown_complete: self._node._shutdown_complete.callback(shutdown_info) # publish management API event yield self.publish( u'{}.on_shutdown'.format(self._uri_prefix), shutdown_info, options=PublishOptions(exclude=details.caller if details else None, acknowledge=True) ) def stop_reactor(): try: self._reactor.stop() except ReactorNotRunning: pass _SHUTDOWN_DELAY = 0.2 self._reactor.callLater(_SHUTDOWN_DELAY, stop_reactor) except: self.log.failure() self._shutdown_requested = False raise else: returnValue(shutdown_info) # to be overridden in derived node classes to shutdown any # specifics (eg clients like for etcd or docker) def _shutdown(self, restart=False, mode=None): pass @wamp.register(None) def get_workers(self, filter_types=[], details=None): """ Returns the list of workers currently running on this node. :param filter_types: :returns: List of worker processes. :rtype: list[dict] """ assert filter_types is None or (type(filter_types) == list and type(ft) == str for ft in filter_types) if filter_types: ft = set(filter_types) worker_ids = [worker_id for worker_id in self._workers if self._workers[worker_id].TYPE in ft] else: worker_ids = self._workers.keys() return sorted(worker_ids) @wamp.register(None) def get_worker(self, worker_id, include_stats=False, details=None): """ Return detailed information about worker. :param worker_id: ID of worker to get information for. :type worker_id: str :param include_stats: If true, include worker run-time statistics. :type include_stats: bool """ if worker_id not in self._workers: emsg = "No worker with ID '{}'".format(worker_id) raise ApplicationError(u'crossbar.error.no_such_worker', emsg) now = datetime.utcnow() worker = self._workers[worker_id] worker_info = { u'id': worker.id, u'pid': worker.pid, u'type': worker.TYPE, u'status': worker.status, u'created': utcstr(worker.created), u'started': utcstr(worker.started), u'startup_time': (worker.started - worker.created).total_seconds() if worker.started else None, u'uptime': (now - worker.started).total_seconds() if worker.started else None, } if include_stats: stats = { u'controller_traffic': worker.get_stats() } worker_info[u'stats'] = stats return worker_info @wamp.register(None) def start_worker(self, worker_id, worker_type, worker_options=None, details=None): """ Start a new worker process in the node. """ self.log.info('Starting {worker_type} worker {worker_id} {worker_klass}', worker_type=worker_type, worker_id=hlid(worker_id), worker_klass=hltype(NodeController.start_worker)) if type(worker_id) != str or worker_id in [u'controller', u'']: raise Exception('invalid worker ID "{}"'.format(worker_id)) if worker_type == u'guest': return self._start_guest_worker(worker_id, worker_options, details=details) elif worker_type in self._node._native_workers: return self._start_native_worker(worker_type, worker_id, worker_options, details=details) else: raise Exception('invalid worker type "{}"'.format(worker_type)) @wamp.register(None) def stop_worker(self, worker_id, kill=False, details=None): """ Stop a running worker. :param worker_id: ID of worker to stop. :type worker_id: str :param kill: If ``True``, kill the process. Otherwise, gracefully shut down the worker. :type kill: bool :returns: Stopping information from the worker. :rtype: dict """ if worker_id not in self._workers: emsg = "No worker with ID '{}'".format(worker_id) raise ApplicationError(u'crossbar.error.no_such_worker', emsg) worker = self._workers[worker_id] if worker.TYPE in self._node._native_workers: return self._stop_native_worker(worker_id, kill=kill, details=details) elif worker.TYPE == u'guest': return self._stop_guest_worker(worker_id, kill=kill, details=details) else: # should not arrive here raise Exception('logic error') @wamp.register(None) def get_worker_log(self, worker_id, limit=100, details=None): """ Get buffered log for a worker. :param worker_id: The worker ID to get log output for. :type worker_id: str :param limit: Limit the amount of log entries returned to the last N entries. :type limit: int :returns: Buffered log for worker. :rtype: list """ if worker_id not in self._workers: emsg = "No worker with ID '{}'".format(worker_id) raise ApplicationError(u'crossbar.error.no_such_worker', emsg) return self._workers[worker_id].getlog(limit) def _start_native_worker(self, worker_type, worker_id, worker_options=None, details=None): # prohibit starting a worker twice # if worker_id in self._workers: emsg = "Could not start worker: a worker with ID '{}' is already running (or starting)".format(worker_id) self.log.error(emsg) raise ApplicationError(u'crossbar.error.worker_already_running', emsg) # check worker options # options = worker_options or {} try: if worker_type in self._node._native_workers: if self._node._native_workers[worker_type]['checkconfig_options']: self._node._native_workers[worker_type]['checkconfig_options'](self.personality, options) else: raise Exception('No checkconfig_options for worker type "{worker_type}" implemented!'.format(worker_type=worker_type)) else: raise Exception('invalid worker type "{}"'.format(worker_type)) except Exception as e: emsg = "Could not start native worker: invalid configuration ({})".format(e) self.log.error(emsg) raise ApplicationError(u'crossbar.error.invalid_configuration', emsg) # the fully qualified worker class as a string worker_class = qual(self._node._native_workers[worker_type]['worker_class']) # allow override Python executable from options # if 'python' in options: exe = options['python'] # the executable must be an absolute path, e.g. /home/oberstet/pypy-2.2.1-linux64/bin/pypy # if not os.path.isabs(exe): emsg = "Invalid worker configuration: python executable '{}' must be an absolute path".format(exe) self.log.error(emsg) raise ApplicationError(u'crossbar.error.invalid_configuration', emsg) # of course the path must exist and actually be executable # if not (os.path.isfile(exe) and os.access(exe, os.X_OK)): emsg = "Invalid worker configuration: python executable '{}' does not exist or isn't an executable".format(exe) self.log.error(emsg) raise ApplicationError(u'crossbar.error.invalid_configuration', emsg) else: exe = sys.executable # allow override default Python module search paths from options # if 'pythonpath' in options: pythonpaths_to_add = [os.path.abspath(os.path.join(self._node._cbdir, p)) for p in options.get('pythonpath', [])] else: pythonpaths_to_add = [] # assemble command line for forking the worker # # all native workers (routers and containers for now) start # from the same script in crossbar/worker/process.py or # from the command "crossbar _exec_worker" when crossbar is # running from a frozen executable (single-file, pyinstaller, etc) # if getattr(sys, 'frozen', False): # if we are inside a frozen crossbar executable, we need to invoke # the crossbar executable with a command ("_exec_worker") args = [exe, self._node.personality.NAME, "_exec_worker"] else: # we are invoking via "-m" so that .pyc files, __pycache__ # etc work properly. this works everywhere, but frozen executables args = [exe, "-u", "-m", "crossbar.worker.main"] args.extend(["--cbdir", self._node._cbdir]) args.extend(["--node", str(self._node._node_id)]) args.extend(["--worker", str(worker_id)]) args.extend(["--realm", self._realm]) args.extend(["--personality", class_name(self._node.personality)]) args.extend(["--klass", worker_class]) args.extend(["--loglevel", get_global_log_level()]) if self._node.options.debug_lifecycle: args.append("--debug-lifecycle") if self._node.options.debug_programflow: args.append("--debug-programflow") if self._node.options.enable_vmprof: args.append("--vmprof") if "shutdown" in options: args.extend(["--shutdown", options["shutdown"]]) # Node-level callback to inject worker arguments # self._node._extend_worker_args(args, options) # allow override worker process title from options # if options.get('title', None): args.extend(['--title', options['title']]) # forward explicit reactor selection # if 'reactor' in options and sys.platform in options['reactor']: args.extend(['--reactor', options['reactor'][sys.platform]]) # FIXME # elif self._node.options.reactor: # args.extend(['--reactor', self._node.options.reactor]) # create worker process environment # worker_env = create_process_env(options) # We need to use the same PYTHONPATH we were started with, so we can # find the Crossbar we're working with -- it may not be the same as the # one on the default path worker_env["PYTHONPATH"] = os.pathsep.join(pythonpaths_to_add + sys.path) # log name of worker # worker_logname = self._node._native_workers[worker_type]['logname'] # each worker is run under its own dedicated WAMP auth role # worker_auth_role = u'crossbar.worker.{}'.format(worker_id) # topic URIs used (later) # starting_topic = self._node._native_workers[worker_type]['topics']['starting'] started_topic = self._node._native_workers[worker_type]['topics']['started'] # add worker tracking instance to the worker map .. # WORKER = self._node._native_workers[worker_type]['class'] worker = WORKER(self, worker_id, details.caller, keeplog=options.get('traceback', None)) self._workers[worker_id] = worker # create a (custom) process endpoint. # if platform.isWindows(): childFDs = None # Use the default Twisted ones else: # The communication between controller and container workers is # using WAMP running over 2 pipes. # For controller->native-worker traffic this runs over FD 0 (`stdin`) # and for the native-worker->controller traffic, this runs over FD 3. # # Note: We use FD 3, not FD 1 (`stdout`) or FD 2 (`stderr`) for # container->controller traffic, so that components running in the # container which happen to write to `stdout` or `stderr` do not # interfere with the container-controller communication. childFDs = {0: "w", 1: "r", 2: "r", 3: "r"} ep = WorkerProcessEndpoint( self._node._reactor, exe, args, env=worker_env, worker=worker, childFDs=childFDs) # ready handling # def on_ready_success(worker_id): self.log.debug('{worker_type} worker "{worker_id}" process {pid} started', worker_type=worker_logname, worker_id=worker.id, pid=worker.pid) self._node._reactor.addSystemEventTrigger( 'before', 'shutdown', self._cleanup_worker, self._node._reactor, worker, ) worker.on_worker_started() started_info = { u'id': worker.id, u'status': worker.status, u'started': utcstr(worker.started), u'who': worker.who, u'pid': worker.pid, u'startup_time': (worker.started - worker.created).total_seconds() if worker.started else None } # FIXME: make start of stats printer dependent on log level .. if False: worker.log_stats(5.) self.publish(started_topic, started_info, options=PublishOptions(exclude=details.caller)) return started_info def on_ready_error(err): del self._workers[worker.id] emsg = 'Failed to start native worker: {}'.format(err.value) self.log.error(emsg) raise ApplicationError(u"crossbar.error.cannot_start", emsg, worker.getlog()) worker.ready.addCallbacks(on_ready_success, on_ready_error) def on_exit_success(_): self.log.info("Node worker {worker.id} ended successfully", worker=worker) # clear worker log worker.log_stats(0) # remove the dedicated node router authrole we dynamically # added for the worker self._node._drop_worker_role(worker_auth_role) # remove our metadata tracking for the worker del self._workers[worker.id] # indicate that the worker excited successfully return True def on_exit_error(err): self.log.info("Node worker {worker.id} ended with error ({err})", worker=worker, err=err) # clear worker log worker.log_stats(0) # remove the dedicated node router authrole we dynamically # added for the worker self._node._drop_worker_role(worker_auth_role) # remove our metadata tracking for the worker del self._workers[worker.id] # indicate that the worker excited with error return False def check_for_shutdown(was_successful): self.log.info('Checking for node shutdown: worker_exit_success={worker_exit_success}, shutdown_requested={shutdown_requested}, node_shutdown_triggers={node_shutdown_triggers}', worker_exit_success=was_successful, shutdown_requested=self._shutdown_requested, node_shutdown_triggers=self._node._node_shutdown_triggers) shutdown = self._shutdown_requested # automatically shutdown node whenever a worker ended (successfully, or with error) # if NODE_SHUTDOWN_ON_WORKER_EXIT in self._node._node_shutdown_triggers: self.log.info("Node worker ended, and trigger '{trigger}' is active: will shutdown node ..", trigger=NODE_SHUTDOWN_ON_WORKER_EXIT) term_print('CROSSBAR:NODE_SHUTDOWN_ON_WORKER_EXIT') shutdown = True # automatically shutdown node when worker ended with error # elif not was_successful and NODE_SHUTDOWN_ON_WORKER_EXIT_WITH_ERROR in self._node._node_shutdown_triggers: self.log.info("Node worker ended with error, and trigger '{trigger}' is active: will shutdown node ..", trigger=NODE_SHUTDOWN_ON_WORKER_EXIT_WITH_ERROR) term_print('CROSSBAR:NODE_SHUTDOWN_ON_WORKER_EXIT_WITH_ERROR') shutdown = True # automatically shutdown node when no more workers are left # elif len(self._workers) == 0 and NODE_SHUTDOWN_ON_LAST_WORKER_EXIT in self._node._node_shutdown_triggers: self.log.info("No more node workers running, and trigger '{trigger}' is active: will shutdown node ..", trigger=NODE_SHUTDOWN_ON_LAST_WORKER_EXIT) term_print('CROSSBAR:NODE_SHUTDOWN_ON_LAST_WORKER_EXIT') shutdown = True # initiate shutdown (but only if we are not already shutting down) # if shutdown: self.shutdown() else: self.log.info('Node will continue to run!') d_on_exit = worker.exit.addCallbacks(on_exit_success, on_exit_error) d_on_exit.addBoth(check_for_shutdown) # create a transport factory for talking WAMP to the native worker # transport_factory = create_native_worker_client_factory(self._node._router_session_factory, worker_auth_role, worker.ready, worker.exit) transport_factory.noisy = False self._workers[worker_id].factory = transport_factory # now (immediately before actually forking) signal the starting of the worker # starting_info = { u'id': worker_id, u'status': worker.status, u'created': utcstr(worker.created), u'who': worker.who, } # the caller gets a progressive result .. if details.progress: details.progress(starting_info) # .. while all others get an event self.publish(starting_topic, starting_info, options=PublishOptions(exclude=details.caller)) # only the following line will actually exec a new worker process - everything before is just setup # for this moment: self.log.debug('Starting new managed worker process for {worker_logname} worker "{worker_id}" using {exe} with args {args}', worker_id=worker_id, worker_logname=worker_logname, exe=exe, args=args) d = ep.connect(transport_factory) def on_connect_success(proto): # this seems to be called immediately when the child process # has been forked. even if it then immediately fails because # e.g. the executable doesn't even exist. in other words, # I'm not sure under what conditions the deferred will errback .. self.log.debug('Native worker "{worker_id}" connected', worker_id=worker_id) worker.on_worker_connected(proto) # dynamically add a dedicated authrole to the router # for the worker we've just started self._node._add_worker_role(worker_auth_role, options) def on_connect_error(err): # not sure when this errback is triggered at all .. self.log.error("Internal error: connection to forked native worker failed ({err})", err=err) # in any case, forward the error .. worker.ready.errback(err) d.addCallbacks(on_connect_success, on_connect_error) return worker.ready @staticmethod def _cleanup_worker(reactor, worker): """ This is called during reactor shutdown and ensures we wait for our subprocesses to shut down nicely. """ log = make_logger() try: log.info("sending TERM to subprocess {pid}", pid=worker.pid) worker.proto.transport.signalProcess('TERM') # wait for the subprocess to shutdown; could add a timeout # after which we send a KILL maybe? d = Deferred() def protocol_closed(_): log.debug("{pid} exited", pid=worker.pid) d.callback(None) # await worker's timely demise worker.exit.addCallback(protocol_closed) def timeout(tried): if d.called: return log.info("waiting for {pid} to exit...", pid=worker.pid) reactor.callLater(1, timeout, tried + 1) if tried > 20: # or just wait forever? log.info("Sending SIGKILL to {pid}", pid=worker.pid) try: worker.proto.transport.signalProcess('KILL') except ProcessExitedAlready: pass # ignore; it's already dead d.callback(None) # or recurse more? timeout(0) return d except ProcessExitedAlready: pass # ignore; it's already dead @inlineCallbacks def _stop_native_worker(self, worker_id, kill, details=None): if worker_id not in self._workers or not isinstance(self._workers[worker_id], NativeWorkerProcess): emsg = "Could not stop native worker: no native worker with ID '{}' currently running".format(worker_id) raise ApplicationError(u'crossbar.error.worker_not_running', emsg) worker = self._workers[worker_id] if worker.status != 'started': emsg = "Could not stop native worker: worker with ID '{}' is not in status 'started', but status: '{}')".format(worker_id, worker.status) raise ApplicationError(u'crossbar.error.worker_not_running', emsg) stop_info = { u'id': worker.id, u'type': worker.TYPE, u'kill': kill, u'who': details.caller if details else None, u'when': utcnow(), } # publish management API event # yield self.publish( u'{}.on_stop_requested'.format(self._uri_prefix), stop_info, options=PublishOptions(exclude=details.caller if details else None, acknowledge=True) ) # send SIGKILL or SIGTERM to worker # if kill: self.log.info("Killing {worker_type} worker with ID '{worker_id}'", worker_type=worker.TYPE, worker_id=worker_id) self._workers[worker_id].proto.transport.signalProcess("KILL") else: self.log.info("Stopping {worker_type} worker with ID '{worker_id}'", worker_type=worker.TYPE, worker_id=worker_id) self._workers[worker_id].factory.stopFactory() self._workers[worker_id].proto.transport.signalProcess('TERM') # wait until the worker is actually done before we return from # this call yield self._workers[worker_id].proto.is_closed returnValue(stop_info) def _start_guest_worker(self, worker_id, worker_config, details=None): """ Start a new guest process on this node. :param config: The guest process configuration. :type config: dict :returns: The PID of the new process. """ # prohibit starting a worker twice # if worker_id in self._workers: emsg = "Could not start worker: a worker with ID '{}' is already running (or starting)".format(worker_id) self.log.error(emsg) raise ApplicationError(u'crossbar.error.worker_already_running', emsg) try: self.personality.check_guest(self.personality, worker_config) except Exception as e: raise ApplicationError(u'crossbar.error.invalid_configuration', 'invalid guest worker configuration: {}'.format(e)) options = worker_config.get('options', {}) # guest process working directory # workdir = self._node._cbdir if 'workdir' in options: workdir = os.path.join(workdir, options['workdir']) workdir = os.path.abspath(workdir) # guest process executable and command line arguments # # first try to configure the fully qualified path for the guest # executable by joining workdir and configured exectuable .. exe = os.path.abspath(os.path.join(workdir, worker_config['executable'])) if check_executable(exe): self.log.info("Using guest worker executable '{exe}' (executable path taken from configuration)", exe=exe) else: # try to detect the fully qualified path for the guest # executable by doing a "which" on the configured executable name exe = which(worker_config['executable']) if exe is not None and check_executable(exe): self.log.info("Using guest worker executable '{exe}' (executable path detected from environment)", exe=exe) else: emsg = "Could not start worker: could not find and executable for '{}'".format(worker_config['executable']) self.log.error(emsg) raise ApplicationError(u'crossbar.error.invalid_configuration', emsg) # guest process command line arguments # args = [exe] args.extend(worker_config.get('arguments', [])) # guest process environment # worker_env = create_process_env(options) # log name of worker # worker_logname = 'Guest' # topic URIs used (later) # starting_topic = u'{}.on_guest_starting'.format(self._uri_prefix) started_topic = u'{}.on_guest_started'.format(self._uri_prefix) # add worker tracking instance to the worker map .. # worker = GuestWorkerProcess(self, worker_id, details.caller, keeplog=options.get('traceback', None)) self._workers[worker_id] = worker # create a (custom) process endpoint # ep = WorkerProcessEndpoint(self._node._reactor, exe, args, path=workdir, env=worker_env, worker=worker) # ready handling # def on_ready_success(proto): self.log.info('{worker_logname} worker "{worker_id}" started', worker_logname=worker_logname, worker_id=worker.id) worker.on_worker_started(proto) self._node._reactor.addSystemEventTrigger( 'before', 'shutdown', self._cleanup_worker, self._node._reactor, worker, ) # directory watcher # if 'watch' in options: if HAS_FS_WATCHER: # assemble list of watched directories watched_dirs = [] for d in options['watch'].get('directories', []): watched_dirs.append(os.path.abspath(os.path.join(self._node._cbdir, d))) worker.watch_timeout = options['watch'].get('timeout', 1) # create a filesystem watcher worker.watcher = FilesystemWatcher(workdir, watched_dirs=watched_dirs) # make sure to stop the watch upon Twisted being shut down def on_shutdown(): worker.watcher.stop() self._node._reactor.addSystemEventTrigger('before', 'shutdown', on_shutdown) # this handler will get fired by the watcher upon detecting an FS event def on_filesystem_change(fs_event): worker.watcher.stop() proto.signal('TERM') if options['watch'].get('action', None) == 'restart': self.log.info("Filesystem watcher detected change {fs_event} - restarting guest in {watch_timeout} seconds ..", fs_event=fs_event, watch_timeout=worker.watch_timeout) # Add a timeout large enough (perhaps add a config option later) self._node._reactor.callLater(worker.watch_timeout, self.start_worker, worker_id, worker_config, details) # Shut the worker down, after the restart event is scheduled # FIXME: all workers should have a stop() method .. # -> 'GuestWorkerProcess' object has no attribute 'stop' # worker.stop() else: self.log.info("Filesystem watcher detected change {fs_event} - no action taken!", fs_event=fs_event) # now start watching .. worker.watcher.start(on_filesystem_change) else: self.log.warn("Cannot watch directories for changes - feature not available") # assemble guest worker startup information # started_info = { u'id': worker.id, u'status': worker.status, u'started': utcstr(worker.started), u'who': worker.who, } self.publish(started_topic, started_info, options=PublishOptions(exclude=details.caller)) return started_info def on_ready_error(err): del self._workers[worker.id] emsg = 'Failed to start guest worker: {}'.format(err.value) self.log.error(emsg) raise ApplicationError(u"crossbar.error.cannot_start", emsg, ep.getlog()) worker.ready.addCallbacks(on_ready_success, on_ready_error) def on_exit_success(res): self.log.info("Guest {worker_id} exited with success", worker_id=worker.id) del self._workers[worker.id] def on_exit_error(err): self.log.error("Guest {worker_id} exited with error {err.value}", worker_id=worker.id, err=err) del self._workers[worker.id] worker.exit.addCallbacks(on_exit_success, on_exit_error) # create a transport factory for talking WAMP to the native worker # transport_factory = create_guest_worker_client_factory(worker_config, worker.ready, worker.exit) transport_factory.noisy = False self._workers[worker_id].factory = transport_factory # now (immediately before actually forking) signal the starting of the worker # starting_info = { u'id': worker_id, u'status': worker.status, u'created': utcstr(worker.created), u'who': worker.who, } # the caller gets a progressive result .. if details.progress: details.progress(starting_info) # .. while all others get an event self.publish(starting_topic, starting_info, options=PublishOptions(exclude=details.caller)) # now actually fork the worker .. # self.log.info('{worker_logname} "{worker_id}" process starting ..', worker_logname=worker_logname, worker_id=worker_id) self.log.debug('{worker_logname} "{worker_id}" process using command line "{cli}" ..', worker_logname=worker_logname, worker_id=worker_id, cli=' '.join(args)) d = ep.connect(transport_factory) def on_connect_success(proto): # this seems to be called immediately when the child process # has been forked. even if it then immediately fails because # e.g. the executable doesn't even exist. in other words, # I'm not sure under what conditions the deferred will # errback - probably only if the forking of a new process fails # at OS level due to out of memory conditions or such. self.log.debug('{worker_logname} "{worker_id}" connected', worker_logname=worker_logname, worker_id=worker_id) # do not comment this: it will lead to on_worker_started being called # _before_ on_worker_connected, and we don't need it! # worker.on_worker_connected(proto) def on_connect_error(err): # not sure when this errback is triggered at all .. see above. self.log.failure( "Internal error: connection to forked guest worker failed ({log_failure.value})", ) # in any case, forward the error .. worker.ready.errback(err) d.addCallbacks(on_connect_success, on_connect_error) return worker.ready def _stop_guest_worker(self, worker_id, kill=False, details=None): """ Stops a currently running guest worker. :param worker_id: The ID of the guest worker to stop. :type worker_id: str """ self.log.debug("stop_guest({worker_id}, kill={kill})", worker_id=worker_id, kill=kill) if worker_id not in self._workers or self._workers[worker_id].TYPE != 'guest': emsg = "Could not stop guest worker: no guest worker with ID '{}' currently running".format(worker_id) raise ApplicationError(u'crossbar.error.worker_not_running', emsg) worker = self._workers[worker_id] stop_info = { u'id': worker.id, u'type': u'guest', u'kill': kill, u'who': details.caller if details else None, u'when': utcnow(), } try: if kill: self._workers[worker_id].proto.transport.signalProcess("KILL") else: self._workers[worker_id].proto.transport.loseConnection() self._workers[worker_id].proto.transport.signalProcess("TERM") except Exception as e: emsg = "Could not stop guest worker with ID '{}': {}".format(worker_id, e) raise ApplicationError(u'crossbar.error.stop_worker_failed', emsg) else: del self._workers[worker_id] return stop_info