def test_basic(self): """ The JSON observer outputs a stream of log events. """ stream = StringIO() observer = _logging.make_JSON_observer(stream) log = make_logger(observer=observer) log.info("Hello") result = stream.getvalue() log_entry = json.loads(result[:-1]) self.assertEqual(result[-1], _logging.record_separator) self.assertEqual(len(log_entry.keys()), 4) self.assertEqual(log_entry["level"], u"info") self.assertEqual(log_entry["text"], u"Hello")
def test_unicode_logs(self): """ Unicode is JSON serialised correctly. """ stream = StringIO() observer = make_JSON_observer(stream) log = make_logger(observer=observer) try: raise Exception("\u2603") except: log.failure("Oh no") result = stream.getvalue() log_entry = json.loads(result[:-1]) self.assertEqual(result[-1], record_separator) self.assertEqual(len(log_entry.keys()), 4) self.assertIn("\u2603", log_entry["text"]) self.assertEqual(log_entry["level"], "critical")
def test_unicode_logs(self): """ Unicode is JSON serialised correctly. """ stream = StringIO() observer = make_JSON_observer(stream) log = make_logger(observer=observer) try: raise Exception(u"\u2603") except: log.failure("Oh no") result = stream.getvalue() log_entry = json.loads(result[:-1]) self.assertEqual(result[-1], record_separator) self.assertEqual(len(log_entry.keys()), 4) self.assertIn(u"\u2603", log_entry["text"]) self.assertEqual(log_entry["level"], u"critical")
def test_failure(self): """ Failures include the stacktrace. """ stream = StringIO() observer = _logging.make_JSON_observer(stream) log = make_logger(observer=observer) try: 1 / 0 except: log.failure("Oh no") result = stream.getvalue() log_entry = json.loads(result[:-1]) self.assertEqual(result[-1], _logging.record_separator) self.assertEqual(len(log_entry.keys()), 4) self.assertIn(u"ZeroDivisionError", log_entry["text"]) self.assertIn(u"Oh no", log_entry["text"]) self.assertEqual(log_entry["level"], u"critical")
def test_raising_during_encoding(self): """ Non-JSON-serialisable parameters are repr()'d. """ stream = StringIO() observer = _logging.make_JSON_observer(stream) log = make_logger(observer=observer) class BadThing(object): def __repr__(self): raise Exception() log.info("hi {obj}", obj=BadThing()) result = stream.getvalue() log_entry = json.loads(result[:-1]) self.assertEqual(result[-1], _logging.record_separator) self.assertEqual(len(log_entry.keys()), 3) self.assertIn(u"MESSAGE LOST", log_entry["text"]) self.assertEqual(log_entry["level"], u"error")
def test_repr_formatting(self): """ Non-JSON-serialisable parameters are repr()'d, and any curly brackets in the result are escaped. """ stream = StringIO() observer = _logging.make_JSON_observer(stream) log = make_logger(observer=observer) class BracketThing(object): def __repr__(self): return "<BracketThing kwargs={}>" log.info("hi {obj}", obj=BracketThing()) result = stream.getvalue() log_entry = json.loads(result[:-1]) self.assertEqual(result[-1], _logging.record_separator) self.assertEqual(len(log_entry.keys()), 5) self.assertEqual(u"hi <BracketThing kwargs={{}}>", log_entry["text"]) self.assertEqual(log_entry["level"], u"info")
def test_not_json_serialisable(self): """ Non-JSON-serialisable parameters are repr()'d. """ stream = StringIO() observer = _logging.make_JSON_observer(stream) log = make_logger(observer=observer) try: 1 / 0 except: log.failure("Oh no", obj=observer) result = stream.getvalue() log_entry = json.loads(result[:-1]) self.assertEqual(result[-1], _logging.record_separator) self.assertEqual(len(log_entry.keys()), 5) self.assertIn(u"ZeroDivisionError", log_entry["text"]) self.assertIn(u"Oh no", log_entry["text"]) self.assertIn(u"<function ", log_entry["obj"]) self.assertEqual(log_entry["level"], u"critical")
def test_raising_during_encoding(self): """ Non-JSON-serialisable parameters are repr()'d, and if that's impossible then the message is lost. """ stream = StringIO() observer = _logging.make_JSON_observer(stream) log = make_logger(observer=observer) class BadThing(object): def __repr__(self): raise Exception() log.info("hi {obj}", obj=BadThing()) result = stream.getvalue() log_entry = json.loads(result[:-1]) self.assertEqual(result[-1], _logging.record_separator) self.assertEqual(len(log_entry.keys()), 3) self.assertIn(u"MESSAGE LOST", log_entry["text"]) self.assertEqual(log_entry["level"], u"error")
def run(): """ Entry point into (native) worker processes. This wires up stuff such that a worker instance is talking WAMP-over-stdio to the node controller. """ import os import sys import platform import signal # Ignore SIGINT so we get consistent behavior on control-C versus # sending SIGINT to the controller process. When the controller is # shutting down, it sends TERM to all its children but ctrl-C # handling will send a SIGINT to all the processes in the group # (so then the controller sends a TERM but the child already or # will very shortly get a SIGINT as well). Twisted installs signal # handlers, but not for SIGINT if there's already a custom one # present. def ignore(sig, frame): log.debug("Ignoring SIGINT in worker.") signal.signal(signal.SIGINT, ignore) # create the top-level parser # import argparse parser = argparse.ArgumentParser() parser.add_argument('-d', '--debug', action='store_true', help='Debug on (optional).') parser.add_argument('--reactor', default=None, choices=['select', 'poll', 'epoll', 'kqueue', 'iocp'], help='Explicit Twisted reactor selection (optional).') parser.add_argument('-c', '--cbdir', type=str, help="Crossbar.io node directory (required).") parser.add_argument('-n', '--node', type=str, help='Crossbar.io node ID (required).') parser.add_argument('-w', '--worker', type=str, help='Crossbar.io worker ID (required).') parser.add_argument('-r', '--realm', type=str, help='Crossbar.io node (management) realm (required).') parser.add_argument('-t', '--type', choices=['router', 'container'], help='Worker type (required).') parser.add_argument('--title', type=str, default=None, help='Worker process title to set (optional).') options = parser.parse_args() # make sure logging to something else than stdio is setup _first_ # from twisted.logger import globalLogBeginner from crossbar._logging import Logger, make_JSON_observer log = Logger() _stderr = sys.stderr flo = make_JSON_observer(_stderr) globalLogBeginner.beginLoggingTo([flo]) try: import setproctitle except ImportError: log.info("Warning: could not set worker process title (setproctitle not installed)") else: # set process title if requested to # if options.title: setproctitle.setproctitle(options.title) else: WORKER_TYPE_TO_TITLE = { 'router': 'crossbar-worker [router]', 'container': 'crossbar-worker [container]' } setproctitle.setproctitle(WORKER_TYPE_TO_TITLE[options.type].strip()) # we use an Autobahn utility to import the "best" available Twisted reactor # from autobahn.twisted.choosereactor import install_reactor reactor = install_reactor(options.reactor) from twisted.python.reflect import qual log.info("Running under {python} using {reactor} reactor", python=platform.python_implementation(), reactor=qual(reactor.__class__).split('.')[-1]) options.cbdir = os.path.abspath(options.cbdir) os.chdir(options.cbdir) # log.msg("Starting from node directory {}".format(options.cbdir)) from crossbar.worker.router import RouterWorkerSession from crossbar.worker.container import ContainerWorkerSession WORKER_TYPE_TO_CLASS = { 'router': RouterWorkerSession, 'container': ContainerWorkerSession } from autobahn.twisted.websocket import WampWebSocketServerProtocol class WorkerServerProtocol(WampWebSocketServerProtocol): def connectionLost(self, reason): try: # this log message is unlikely to reach the controller (unless # only stdin/stdout pipes were lost, but not stderr) log.warn("Connection to node controller lost.") WampWebSocketServerProtocol.connectionLost(self, reason) except: pass finally: # loosing the connection to the node controller is fatal: # stop the reactor and exit with error if reactor.running: reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1) reactor.stop() # if the reactor *isn't* running, we're already shutting down try: # create a WAMP application session factory # from autobahn.twisted.wamp import ApplicationSessionFactory from autobahn.wamp.types import ComponentConfig session_config = ComponentConfig(realm=options.realm, extra=options) session_factory = ApplicationSessionFactory(session_config) session_factory.session = WORKER_TYPE_TO_CLASS[options.type] # create a WAMP-over-WebSocket transport server factory # from autobahn.twisted.websocket import WampWebSocketServerFactory transport_factory = WampWebSocketServerFactory(session_factory, "ws://localhost", debug=False, debug_wamp=False) transport_factory.protocol = WorkerServerProtocol transport_factory.setProtocolOptions(failByDrop=False) # create a protocol instance and wire up to stdio # from twisted.internet import stdio proto = transport_factory.buildProtocol(None) stdio.StandardIO(proto) # now start reactor loop # log.info("Entering event loop...") reactor.run() except Exception as e: log.info("Unhandled exception: {}".format(e)) if reactor.running: reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1) reactor.stop() else: sys.exit(1)
def run(options, reactor=None): """ Entry point into (native) worker processes. This wires up stuff such that a worker instance is talking WAMP-over-stdio to the node controller. """ import os import sys import platform import signal # make sure logging to something else than stdio is setup _first_ # from crossbar._logging import make_JSON_observer, cb_logging_aware from txaio import make_logger, start_logging from twisted.logger import globalLogPublisher from twisted.python.reflect import qual log = make_logger() # Print a magic phrase that tells the capturing logger that it supports # Crossbar's rich logging print(cb_logging_aware, file=sys.__stderr__) sys.__stderr__.flush() flo = make_JSON_observer(sys.__stderr__) globalLogPublisher.addObserver(flo) # Ignore SIGINT so we get consistent behavior on control-C versus # sending SIGINT to the controller process. When the controller is # shutting down, it sends TERM to all its children but ctrl-C # handling will send a SIGINT to all the processes in the group # (so then the controller sends a TERM but the child already or # will very shortly get a SIGINT as well). Twisted installs signal # handlers, but not for SIGINT if there's already a custom one # present. def ignore(sig, frame): log.debug("Ignoring SIGINT in worker.") signal.signal(signal.SIGINT, ignore) # actually begin logging start_logging(None, options.loglevel) # we use an Autobahn utility to import the "best" available Twisted reactor # from autobahn.twisted.choosereactor import install_reactor reactor = install_reactor(options.reactor) # eg: crossbar.worker.container.ContainerWorkerSession l = options.klass.split('.') worker_module, worker_klass = '.'.join(l[:-1]), l[-1] # now load the worker module and class _mod = importlib.import_module(worker_module) klass = getattr(_mod, worker_klass) log.info( 'Started {worker_title} worker "{worker_id}" on node "{node_id}" [{klass} / {python}-{reactor}]', worker_title=klass.WORKER_TITLE, klass=options.klass, node_id=options.node, worker_id=options.worker, pid=os.getpid(), python=platform.python_implementation(), reactor=qual(reactor.__class__).split('.')[-1], ) # set process title if requested to # try: import setproctitle except ImportError: log.debug( "Could not set worker process title (setproctitle not installed)") else: if options.title: setproctitle.setproctitle(options.title) else: setproctitle.setproctitle('crossbar-worker [{}]'.format( options.klass)) # node directory # options.cbdir = os.path.abspath(options.cbdir) os.chdir(options.cbdir) # log.msg("Starting from node directory {}".format(options.cbdir)) # set process title if requested to # try: import setproctitle except ImportError: log.debug( "Could not set worker process title (setproctitle not installed)") else: if options.title: setproctitle.setproctitle(options.title) else: setproctitle.setproctitle('crossbar-worker [{}]'.format( options.klass)) from twisted.internet.error import ConnectionDone from autobahn.twisted.websocket import WampWebSocketServerProtocol class WorkerServerProtocol(WampWebSocketServerProtocol): def connectionLost(self, reason): # the behavior here differs slightly whether we're shutting down orderly # or shutting down because of "issues" if isinstance(reason.value, ConnectionDone): was_clean = True else: was_clean = False try: # this log message is unlikely to reach the controller (unless # only stdin/stdout pipes were lost, but not stderr) if was_clean: log.info("Connection to node controller closed cleanly") else: log.warn("Connection to node controller lost: {reason}", reason=reason) # give the WAMP transport a change to do it's thing WampWebSocketServerProtocol.connectionLost(self, reason) except: # we're in the process of shutting down .. so ignore .. pass finally: # after the connection to the node controller is gone, # the worker is "orphane", and should exit # determine process exit code if was_clean: exit_code = 0 else: exit_code = 1 # exit the whole worker process when the reactor has stopped reactor.addSystemEventTrigger('after', 'shutdown', os._exit, exit_code) # stop the reactor try: reactor.stop() except ReactorNotRunning: pass try: # create a WAMP application session factory # from autobahn.twisted.wamp import ApplicationSessionFactory from autobahn.wamp.types import ComponentConfig session_config = ComponentConfig(realm=options.realm, extra=options) session_factory = ApplicationSessionFactory(session_config) session_factory.session = klass # create a WAMP-over-WebSocket transport server factory # from autobahn.twisted.websocket import WampWebSocketServerFactory transport_factory = WampWebSocketServerFactory(session_factory, u'ws://localhost') transport_factory.protocol = WorkerServerProtocol transport_factory.setProtocolOptions(failByDrop=False) # create a protocol instance and wire up to stdio # from twisted.python.runtime import platform as _platform from twisted.internet import stdio proto = transport_factory.buildProtocol(None) if _platform.isWindows(): stdio.StandardIO(proto) else: stdio.StandardIO(proto, stdout=3) # now start reactor loop # if False: log.info("vmprof enabled.") import os import vmprof PROFILE_FILE = 'vmprof_{}.dat'.format(os.getpid()) outfd = os.open(PROFILE_FILE, os.O_RDWR | os.O_CREAT | os.O_TRUNC) vmprof.enable(outfd, period=0.01) log.info("Entering event loop...") reactor.run() vmprof.disable() else: log.debug("Entering event loop...") reactor.run() except Exception as e: log.info("Unhandled exception: {e}", e=e) if reactor.running: reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1) reactor.stop() else: sys.exit(1)
def _run_command_exec_worker(options, reactor=None, personality=None): """ Entry point into (native) worker processes. This wires up stuff such that a worker instance is talking WAMP-over-stdio to the node controller. """ import os import sys import platform import signal # https://coverage.readthedocs.io/en/coverage-4.4.2/subprocess.html#measuring-sub-processes MEASURING_COVERAGE = False if 'COVERAGE_PROCESS_START' in os.environ: try: import coverage except ImportError: pass else: # The following will read the environment variable COVERAGE_PROCESS_START, # and that should be set to the .coveragerc file: # # export COVERAGE_PROCESS_START=${PWD}/.coveragerc # coverage.process_startup() MEASURING_COVERAGE = True # we use an Autobahn utility to import the "best" available Twisted reactor from autobahn.twisted.choosereactor import install_reactor reactor = install_reactor(options.reactor) # make sure logging to something else than stdio is setup _first_ from crossbar._logging import make_JSON_observer, cb_logging_aware from txaio import make_logger, start_logging from twisted.logger import globalLogPublisher from twisted.python.reflect import qual log = make_logger() # Print a magic phrase that tells the capturing logger that it supports # Crossbar's rich logging print(cb_logging_aware, file=sys.__stderr__) sys.__stderr__.flush() flo = make_JSON_observer(sys.__stderr__) globalLogPublisher.addObserver(flo) # Ignore SIGINT so we get consistent behavior on control-C versus # sending SIGINT to the controller process. When the controller is # shutting down, it sends TERM to all its children but ctrl-C # handling will send a SIGINT to all the processes in the group # (so then the controller sends a TERM but the child already or # will very shortly get a SIGINT as well). Twisted installs signal # handlers, but not for SIGINT if there's already a custom one # present. def ignore(sig, frame): log.debug("Ignoring SIGINT in worker.") signal.signal(signal.SIGINT, ignore) # actually begin logging start_logging(None, options.loglevel) # get personality klass, eg "crossbar.personality.Personality" l = options.personality.split('.') personality_module, personality_klass = '.'.join(l[:-1]), l[-1] # now load the personality module and class _mod = importlib.import_module(personality_module) Personality = getattr(_mod, personality_klass) # get worker klass, eg "crossbar.worker.container.ContainerController" l = options.klass.split('.') worker_module, worker_klass = '.'.join(l[:-1]), l[-1] # now load the worker module and class _mod = importlib.import_module(worker_module) klass = getattr(_mod, worker_klass) log.info( 'Starting worker "{worker_id}" for node "{node_id}" with personality "{personality}" {worker_class}', worker_id=options.worker, node_id=options.node, personality=Personality.NAME, worker_class=hltype(klass), ) log.info( 'Running as PID {pid} on {python}-{reactor}', pid=os.getpid(), python=platform.python_implementation(), reactor=qual(reactor.__class__).split('.')[-1], ) if MEASURING_COVERAGE: log.info(hl('Code coverage measurements enabled (coverage={coverage_version}).', color='green', bold=True), coverage_version=coverage.__version__) # set process title if requested to # try: import setproctitle except ImportError: log.debug("Could not set worker process title (setproctitle not installed)") else: if options.title: setproctitle.setproctitle(options.title) else: setproctitle.setproctitle('crossbar-worker [{}]'.format(options.klass)) # node directory # options.cbdir = os.path.abspath(options.cbdir) os.chdir(options.cbdir) # log.msg("Starting from node directory {}".format(options.cbdir)) # set process title if requested to # try: import setproctitle except ImportError: log.debug("Could not set worker process title (setproctitle not installed)") else: if options.title: setproctitle.setproctitle(options.title) else: setproctitle.setproctitle( 'crossbar-worker [{}]'.format(options.klass) ) from twisted.internet.error import ConnectionDone from autobahn.twisted.websocket import WampWebSocketServerProtocol class WorkerServerProtocol(WampWebSocketServerProtocol): def connectionLost(self, reason): # the behavior here differs slightly whether we're shutting down orderly # or shutting down because of "issues" if isinstance(reason.value, ConnectionDone): was_clean = True else: was_clean = False try: # this log message is unlikely to reach the controller (unless # only stdin/stdout pipes were lost, but not stderr) if was_clean: log.info("Connection to node controller closed cleanly") else: log.warn("Connection to node controller lost: {reason}", reason=reason) # give the WAMP transport a change to do it's thing WampWebSocketServerProtocol.connectionLost(self, reason) except: # we're in the process of shutting down .. so ignore .. pass finally: # after the connection to the node controller is gone, # the worker is "orphane", and should exit # determine process exit code if was_clean: exit_code = 0 else: exit_code = 1 # exit the whole worker process when the reactor has stopped reactor.addSystemEventTrigger('after', 'shutdown', os._exit, exit_code) # stop the reactor try: reactor.stop() except ReactorNotRunning: pass try: # define a WAMP application session factory # from autobahn.wamp.types import ComponentConfig def make_session(): session_config = ComponentConfig(realm=options.realm, extra=options) session = klass(config=session_config, reactor=reactor, personality=Personality) return session # create a WAMP-over-WebSocket transport server factory # from autobahn.twisted.websocket import WampWebSocketServerFactory transport_factory = WampWebSocketServerFactory(make_session, u'ws://localhost') transport_factory.protocol = WorkerServerProtocol transport_factory.setProtocolOptions(failByDrop=False) # create a protocol instance and wire up to stdio # from twisted.python.runtime import platform as _platform from twisted.internet import stdio proto = transport_factory.buildProtocol(None) if _platform.isWindows(): stdio.StandardIO(proto) else: stdio.StandardIO(proto, stdout=3) # now start reactor loop # if False: log.info("vmprof enabled.") import os import vmprof PROFILE_FILE = 'vmprof_{}.dat'.format(os.getpid()) outfd = os.open(PROFILE_FILE, os.O_RDWR | os.O_CREAT | os.O_TRUNC) vmprof.enable(outfd, period=0.01) log.info(hl('Entering event reactor ...', color='cyan', bold=True)) reactor.run() vmprof.disable() else: log.info(hl('Entering event reactor ...', color='cyan', bold=True)) reactor.run() except Exception as e: log.info("Unhandled exception: {e}", e=e) if reactor.running: reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1) reactor.stop() else: sys.exit(1)
def _run_command_exec_worker(options, reactor=None, personality=None): """ Entry point into (native) worker processes. This wires up stuff such that a worker instance is talking WAMP-over-stdio to the node controller. """ import os import sys import platform import signal # https://coverage.readthedocs.io/en/coverage-4.4.2/subprocess.html#measuring-sub-processes MEASURING_COVERAGE = False if 'COVERAGE_PROCESS_START' in os.environ: try: import coverage except ImportError: pass else: # The following will read the environment variable COVERAGE_PROCESS_START, # and that should be set to the .coveragerc file: # # export COVERAGE_PROCESS_START=${PWD}/.coveragerc # coverage.process_startup() MEASURING_COVERAGE = True # we use an Autobahn utility to import the "best" available Twisted reactor from autobahn.twisted.choosereactor import install_reactor reactor = install_reactor(options.reactor) # make sure logging to something else than stdio is setup _first_ from crossbar._logging import make_JSON_observer, cb_logging_aware from txaio import make_logger, start_logging from twisted.logger import globalLogPublisher from twisted.python.reflect import qual log = make_logger() # Print a magic phrase that tells the capturing logger that it supports # Crossbar's rich logging print(cb_logging_aware, file=sys.__stderr__) sys.__stderr__.flush() flo = make_JSON_observer(sys.__stderr__) globalLogPublisher.addObserver(flo) # Ignore SIGINT so we get consistent behavior on control-C versus # sending SIGINT to the controller process. When the controller is # shutting down, it sends TERM to all its children but ctrl-C # handling will send a SIGINT to all the processes in the group # (so then the controller sends a TERM but the child already or # will very shortly get a SIGINT as well). Twisted installs signal # handlers, but not for SIGINT if there's already a custom one # present. def ignore(sig, frame): log.debug("Ignoring SIGINT in worker.") signal.signal(signal.SIGINT, ignore) # actually begin logging start_logging(None, options.loglevel) # now check if we can import the requested personality Personality = crossbar.personalities().get(options.personality, None) if not Personality: raise Exception('logic error: personality "{}" not installed'.format(options.personality)) # eg: crossbar.worker.container.ContainerController l = options.klass.split('.') worker_module, worker_klass = '.'.join(l[:-1]), l[-1] # now load the worker module and class _mod = importlib.import_module(worker_module) klass = getattr(_mod, worker_klass) log.info( 'Starting worker "{worker_id}" for node "{node_id}" with personality "{personality}" {worker_class}', worker_id=options.worker, node_id=options.node, personality=Personality.NAME, worker_class=hltype(klass), ) log.info( 'Running as PID {pid} on {python}-{reactor}', pid=os.getpid(), python=platform.python_implementation(), reactor=qual(reactor.__class__).split('.')[-1], ) if MEASURING_COVERAGE: log.info(hl('Code coverage measurements enabled (coverage={coverage_version}).', color='green', bold=True), coverage_version=coverage.__version__) # set process title if requested to # try: import setproctitle except ImportError: log.debug("Could not set worker process title (setproctitle not installed)") else: if options.title: setproctitle.setproctitle(options.title) else: setproctitle.setproctitle('crossbar-worker [{}]'.format(options.klass)) # node directory # options.cbdir = os.path.abspath(options.cbdir) os.chdir(options.cbdir) # log.msg("Starting from node directory {}".format(options.cbdir)) # set process title if requested to # try: import setproctitle except ImportError: log.debug("Could not set worker process title (setproctitle not installed)") else: if options.title: setproctitle.setproctitle(options.title) else: setproctitle.setproctitle( 'crossbar-worker [{}]'.format(options.klass) ) from twisted.internet.error import ConnectionDone from autobahn.twisted.websocket import WampWebSocketServerProtocol class WorkerServerProtocol(WampWebSocketServerProtocol): def connectionLost(self, reason): # the behavior here differs slightly whether we're shutting down orderly # or shutting down because of "issues" if isinstance(reason.value, ConnectionDone): was_clean = True else: was_clean = False try: # this log message is unlikely to reach the controller (unless # only stdin/stdout pipes were lost, but not stderr) if was_clean: log.info("Connection to node controller closed cleanly") else: log.warn("Connection to node controller lost: {reason}", reason=reason) # give the WAMP transport a change to do it's thing WampWebSocketServerProtocol.connectionLost(self, reason) except: # we're in the process of shutting down .. so ignore .. pass finally: # after the connection to the node controller is gone, # the worker is "orphane", and should exit # determine process exit code if was_clean: exit_code = 0 else: exit_code = 1 # exit the whole worker process when the reactor has stopped reactor.addSystemEventTrigger('after', 'shutdown', os._exit, exit_code) # stop the reactor try: reactor.stop() except ReactorNotRunning: pass try: # define a WAMP application session factory # from autobahn.wamp.types import ComponentConfig def make_session(): session_config = ComponentConfig(realm=options.realm, extra=options) session = klass(config=session_config, reactor=reactor, personality=Personality) return session # create a WAMP-over-WebSocket transport server factory # from autobahn.twisted.websocket import WampWebSocketServerFactory transport_factory = WampWebSocketServerFactory(make_session, u'ws://localhost') transport_factory.protocol = WorkerServerProtocol transport_factory.setProtocolOptions(failByDrop=False) # create a protocol instance and wire up to stdio # from twisted.python.runtime import platform as _platform from twisted.internet import stdio proto = transport_factory.buildProtocol(None) if _platform.isWindows(): stdio.StandardIO(proto) else: stdio.StandardIO(proto, stdout=3) # now start reactor loop # if False: log.info("vmprof enabled.") import os import vmprof PROFILE_FILE = 'vmprof_{}.dat'.format(os.getpid()) outfd = os.open(PROFILE_FILE, os.O_RDWR | os.O_CREAT | os.O_TRUNC) vmprof.enable(outfd, period=0.01) log.info(hl('Entering event reactor ...', color='cyan', bold=True)) reactor.run() vmprof.disable() else: log.info(hl('Entering event reactor ...', color='cyan', bold=True)) reactor.run() except Exception as e: log.info("Unhandled exception: {e}", e=e) if reactor.running: reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1) reactor.stop() else: sys.exit(1)
def _run_command_exec_worker(options, reactor=None, personality=None): """ Entry point into (native) worker processes. This wires up stuff such that a worker instance is talking WAMP-over-stdio to the node controller. """ import os import sys import platform import signal # https://coverage.readthedocs.io/en/coverage-4.4.2/subprocess.html#measuring-sub-processes MEASURING_COVERAGE = False if 'COVERAGE_PROCESS_START' in os.environ: try: import coverage except ImportError: pass else: # The following will read the environment variable COVERAGE_PROCESS_START, # and that should be set to the .coveragerc file: # # export COVERAGE_PROCESS_START=${PWD}/.coveragerc # coverage.process_startup() MEASURING_COVERAGE = True # we use an Autobahn utility to import the "best" available Twisted reactor from autobahn.twisted.choosereactor import install_reactor reactor = install_reactor(options.reactor) # make sure logging to something else than stdio is setup _first_ from crossbar._logging import make_JSON_observer, cb_logging_aware from txaio import make_logger, start_logging from twisted.logger import globalLogPublisher from twisted.python.reflect import qual log = make_logger() # Print a magic phrase that tells the capturing logger that it supports # Crossbar's rich logging print(cb_logging_aware, file=sys.__stderr__) sys.__stderr__.flush() flo = make_JSON_observer(sys.__stderr__) globalLogPublisher.addObserver(flo) term_print('CROSSBAR[{}]:WORKER_STARTING'.format(options.worker)) # Ignore SIGINT so we get consistent behavior on control-C versus # sending SIGINT to the controller process. When the controller is # shutting down, it sends TERM to all its children but ctrl-C # handling will send a SIGINT to all the processes in the group # (so then the controller sends a TERM but the child already or # will very shortly get a SIGINT as well). Twisted installs signal # handlers, but not for SIGINT if there's already a custom one # present. def ignore(sig, frame): log.debug("Ignoring SIGINT in worker.") signal.signal(signal.SIGINT, ignore) # actually begin logging start_logging(None, options.loglevel) # get personality klass, eg "crossbar.personality.Personality" l = options.personality.split('.') personality_module, personality_klass = '.'.join(l[:-1]), l[-1] # now load the personality module and class _mod = importlib.import_module(personality_module) Personality = getattr(_mod, personality_klass) # get worker klass, eg "crossbar.worker.container.ContainerController" l = options.klass.split('.') worker_module, worker_klass = '.'.join(l[:-1]), l[-1] # now load the worker module and class _mod = importlib.import_module(worker_module) klass = getattr(_mod, worker_klass) log.info( 'Starting {worker_type}-worker "{worker_id}" on node "{node_id}" (personality "{personality}") and local node management realm "{realm}" .. {worker_class}', worker_type=hl(klass.WORKER_TYPE), worker_id=hlid(options.worker), node_id=hlid(options.node), realm=hlid(options.realm), personality=hl(Personality.NAME), worker_class=hltype(klass), ) log.info( 'Running as PID {pid} on {python}-{reactor}', pid=os.getpid(), python=platform.python_implementation(), reactor=qual(reactor.__class__).split('.')[-1], ) if MEASURING_COVERAGE: log.info(hl( 'Code coverage measurements enabled (coverage={coverage_version}).', color='green', bold=True), coverage_version=coverage.__version__) # set process title if requested to # try: import setproctitle except ImportError: log.debug( "Could not set worker process title (setproctitle not installed)") else: if options.title: setproctitle.setproctitle(options.title) else: setproctitle.setproctitle('crossbar-worker [{}]'.format( options.klass)) # node directory # options.cbdir = os.path.abspath(options.cbdir) os.chdir(options.cbdir) # log.msg("Starting from node directory {}".format(options.cbdir)) # set process title if requested to # try: import setproctitle except ImportError: log.debug( "Could not set worker process title (setproctitle not installed)") else: if options.title: setproctitle.setproctitle(options.title) else: setproctitle.setproctitle('crossbar-worker [{}]'.format( options.klass)) from twisted.internet.error import ConnectionDone from autobahn.twisted.websocket import WampWebSocketServerProtocol class WorkerServerProtocol(WampWebSocketServerProtocol): def connectionLost(self, reason): # the behavior here differs slightly whether we're shutting down orderly # or shutting down because of "issues" if isinstance(reason.value, ConnectionDone): was_clean = True else: was_clean = False try: # this log message is unlikely to reach the controller (unless # only stdin/stdout pipes were lost, but not stderr) if was_clean: log.info("Connection to node controller closed cleanly") else: log.warn("Connection to node controller lost: {reason}", reason=reason) # give the WAMP transport a change to do it's thing WampWebSocketServerProtocol.connectionLost(self, reason) except: # we're in the process of shutting down .. so ignore .. pass finally: # after the connection to the node controller is gone, # the worker is "orphane", and should exit # determine process exit code if was_clean: exit_code = 0 else: exit_code = 1 # exit the whole worker process when the reactor has stopped reactor.addSystemEventTrigger('after', 'shutdown', os._exit, exit_code) # stop the reactor try: reactor.stop() except ReactorNotRunning: pass # if vmprof global profiling is enabled via command line option, this will carry # the file where vmprof writes its profile data if _HAS_VMPROF: _vm_prof = { # need to put this into a dict, since FDs are ints, and python closures can't # write to this otherwise 'outfd': None } # https://twistedmatrix.com/documents/current/api/twisted.internet.interfaces.IReactorCore.html # Each "system event" in Twisted, such as 'startup', 'shutdown', and 'persist', has 3 phases: # 'before', 'during', and 'after' (in that order, of course). These events will be fired # internally by the Reactor. def before_reactor_started(): term_print('CROSSBAR[{}]:REACTOR_STARTING'.format(options.worker)) def after_reactor_started(): term_print('CROSSBAR[{}]:REACTOR_STARTED'.format(options.worker)) if _HAS_VMPROF and options.vmprof: outfn = os.path.join( options.cbdir, '.vmprof-worker-{}-{}.dat'.format(options.worker, os.getpid())) _vm_prof['outfd'] = os.open(outfn, os.O_RDWR | os.O_CREAT | os.O_TRUNC) vmprof.enable(_vm_prof['outfd'], period=0.01) term_print('CROSSBAR[{}]:VMPROF_ENABLED:{}'.format( options.worker, outfn)) def before_reactor_stopped(): term_print('CROSSBAR[{}]:REACTOR_STOPPING'.format(options.worker)) if _HAS_VMPROF and options.vmprof and _vm_prof['outfd']: vmprof.disable() term_print('CROSSBAR[{}]:VMPROF_DISABLED'.format(options.worker)) def after_reactor_stopped(): # FIXME: we are indeed reaching this line, however, # the log output does not work (it also doesnt work using # plain old print). Dunno why. # my theory about this issue is: by the time this line # is reached, Twisted has already closed the stdout/stderr # pipes. hence we do an evil trick: we directly write to # the process' controlling terminal # https://unix.stackexchange.com/a/91716/52500 term_print('CROSSBAR[{}]:REACTOR_STOPPED'.format(options.worker)) reactor.addSystemEventTrigger('before', 'startup', before_reactor_started) reactor.addSystemEventTrigger('after', 'startup', after_reactor_started) reactor.addSystemEventTrigger('before', 'shutdown', before_reactor_stopped) reactor.addSystemEventTrigger('after', 'shutdown', after_reactor_stopped) try: # define a WAMP application session factory # from autobahn.wamp.types import ComponentConfig def make_session(): session_config = ComponentConfig(realm=options.realm, extra=options) session = klass(config=session_config, reactor=reactor, personality=Personality) return session # create a WAMP-over-WebSocket transport server factory # from autobahn.twisted.websocket import WampWebSocketServerFactory transport_factory = WampWebSocketServerFactory(make_session, 'ws://localhost') transport_factory.protocol = WorkerServerProtocol transport_factory.setProtocolOptions(failByDrop=False) # create a protocol instance and wire up to stdio # from twisted.python.runtime import platform as _platform from twisted.internet import stdio proto = transport_factory.buildProtocol(None) if _platform.isWindows(): stdio.StandardIO(proto) else: stdio.StandardIO(proto, stdout=3) # now start reactor loop # log.info(hl('Entering event reactor ...', color='green', bold=True)) reactor.run() except Exception as e: log.info("Unhandled exception: {e}", e=e) if reactor.running: reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1) reactor.stop() else: sys.exit(1)
def run(): """ Entry point into (native) worker processes. This wires up stuff such that a worker instance is talking WAMP-over-stdio to the node controller. """ import os import sys import platform import signal # Ignore SIGINT so we get consistent behavior on control-C versus # sending SIGINT to the controller process. When the controller is # shutting down, it sends TERM to all its children but ctrl-C # handling will send a SIGINT to all the processes in the group # (so then the controller sends a TERM but the child already or # will very shortly get a SIGINT as well). Twisted installs signal # handlers, but not for SIGINT if there's already a custom one # present. def ignore(sig, frame): log.debug("Ignoring SIGINT in worker.") signal.signal(signal.SIGINT, ignore) # create the top-level parser # import argparse parser = argparse.ArgumentParser() parser.add_argument('--reactor', default=None, choices=['select', 'poll', 'epoll', 'kqueue', 'iocp'], help='Explicit Twisted reactor selection (optional).') parser.add_argument('--loglevel', default="info", choices=['none', 'error', 'warn', 'info', 'debug', 'trace'], help='Initial log level.') parser.add_argument('-c', '--cbdir', type=str, help="Crossbar.io node directory (required).") parser.add_argument('-n', '--node', type=str, help='Crossbar.io node ID (required).') parser.add_argument('-w', '--worker', type=str, help='Crossbar.io worker ID (required).') parser.add_argument('-r', '--realm', type=str, help='Crossbar.io node (management) realm (required).') parser.add_argument('-t', '--type', choices=['router', 'container'], help='Worker type (required).') parser.add_argument('--title', type=str, default=None, help='Worker process title to set (optional).') options = parser.parse_args() # make sure logging to something else than stdio is setup _first_ # from crossbar._logging import make_JSON_observer, cb_logging_aware, _stderr from crossbar._logging import make_logger, log_publisher, start_logging from crossbar._logging import set_global_log_level # Set the global log level set_global_log_level(options.loglevel) log = make_logger() # Print a magic phrase that tells the capturing logger that it supports # Crossbar's rich logging print(cb_logging_aware, file=_stderr) _stderr.flush() flo = make_JSON_observer(_stderr) log_publisher.addObserver(flo) start_logging() try: import setproctitle except ImportError: log.debug("Could not set worker process title (setproctitle not installed)") else: # set process title if requested to # if options.title: setproctitle.setproctitle(options.title) else: WORKER_TYPE_TO_TITLE = { 'router': 'crossbar-worker [router]', 'container': 'crossbar-worker [container]' } setproctitle.setproctitle(WORKER_TYPE_TO_TITLE[options.type].strip()) # we use an Autobahn utility to import the "best" available Twisted reactor # from autobahn.twisted.choosereactor import install_reactor reactor = install_reactor(options.reactor) from twisted.python.reflect import qual log.info("Worker running under {python}-{reactor}", python=platform.python_implementation(), reactor=qual(reactor.__class__).split('.')[-1]) options.cbdir = os.path.abspath(options.cbdir) os.chdir(options.cbdir) # log.msg("Starting from node directory {}".format(options.cbdir)) from crossbar.worker.router import RouterWorkerSession from crossbar.worker.container import ContainerWorkerSession WORKER_TYPE_TO_CLASS = { 'router': RouterWorkerSession, 'container': ContainerWorkerSession } from autobahn.twisted.websocket import WampWebSocketServerProtocol class WorkerServerProtocol(WampWebSocketServerProtocol): def connectionLost(self, reason): try: # this log message is unlikely to reach the controller (unless # only stdin/stdout pipes were lost, but not stderr) log.warn("Connection to node controller lost.") WampWebSocketServerProtocol.connectionLost(self, reason) except: pass finally: # losing the connection to the node controller is fatal: # stop the reactor and exit with error log.info("No more controller connection; shutting down.") reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1) try: reactor.stop() except ReactorNotRunning: pass try: # create a WAMP application session factory # from autobahn.twisted.wamp import ApplicationSessionFactory from autobahn.wamp.types import ComponentConfig session_config = ComponentConfig(realm=options.realm, extra=options) session_factory = ApplicationSessionFactory(session_config) session_factory.session = WORKER_TYPE_TO_CLASS[options.type] # create a WAMP-over-WebSocket transport server factory # from autobahn.twisted.websocket import WampWebSocketServerFactory transport_factory = WampWebSocketServerFactory(session_factory, "ws://localhost", debug=False, debug_wamp=False) transport_factory.protocol = WorkerServerProtocol transport_factory.setProtocolOptions(failByDrop=False) # create a protocol instance and wire up to stdio # from twisted.python.runtime import platform as _platform from twisted.internet import stdio proto = transport_factory.buildProtocol(None) if _platform.isWindows(): stdio.StandardIO(proto) else: stdio.StandardIO(proto, stdout=3) # now start reactor loop # if False: log.info("vmprof enabled.") import os import vmprof PROFILE_FILE = 'vmprof_{}.dat'.format(os.getpid()) outfd = os.open(PROFILE_FILE, os.O_RDWR | os.O_CREAT | os.O_TRUNC) vmprof.enable(outfd, period=0.01) log.info("Entering event loop...") reactor.run() vmprof.disable() else: log.debug("Entering event loop...") reactor.run() except Exception as e: log.info("Unhandled exception: {}".format(e)) if reactor.running: reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1) reactor.stop() else: sys.exit(1)
def run(): """ Entry point into (native) worker processes. This wires up stuff such that a worker instance is talking WAMP-over-stdio to the node controller. """ import os import sys import platform import signal # Ignore SIGINT so we get consistent behavior on control-C versus # sending SIGINT to the controller process. When the controller is # shutting down, it sends TERM to all its children but ctrl-C # handling will send a SIGINT to all the processes in the group # (so then the controller sends a TERM but the child already or # will very shortly get a SIGINT as well). Twisted installs signal # handlers, but not for SIGINT if there's already a custom one # present. def ignore(sig, frame): log.debug("Ignoring SIGINT in worker.") signal.signal(signal.SIGINT, ignore) # create the top-level parser # import argparse parser = argparse.ArgumentParser() parser.add_argument('--reactor', default=None, choices=['select', 'poll', 'epoll', 'kqueue', 'iocp'], help='Explicit Twisted reactor selection (optional).') parser.add_argument( '--loglevel', default="info", choices=['none', 'error', 'warn', 'info', 'debug', 'trace'], help='Initial log level.') parser.add_argument('-c', '--cbdir', type=six.text_type, help="Crossbar.io node directory (required).") parser.add_argument('-r', '--realm', type=six.text_type, help='Crossbar.io node (management) realm (required).') parser.add_argument('-t', '--type', choices=['router', 'container', 'websocket-testee'], help='Worker type (required).') parser.add_argument('-w', '--worker', type=six.text_type, help='Crossbar.io worker ID (required).') parser.add_argument('--title', type=six.text_type, default=None, help='Worker process title to set (optional).') parser.add_argument( '--expose_controller', type=bool, default=False, help= 'Expose node controller session to all components (this feature requires Crossbar.io Fabric extension).' ) parser.add_argument( '--expose_shared', type=bool, default=False, help= 'Expose a shared object to all components (this feature requires Crossbar.io Fabric extension).' ) options = parser.parse_args() # make sure logging to something else than stdio is setup _first_ # from crossbar._logging import make_JSON_observer, cb_logging_aware from txaio import make_logger, start_logging from twisted.logger import globalLogPublisher log = make_logger() # Print a magic phrase that tells the capturing logger that it supports # Crossbar's rich logging print(cb_logging_aware, file=sys.__stderr__) sys.__stderr__.flush() flo = make_JSON_observer(sys.__stderr__) globalLogPublisher.addObserver(flo) start_logging(None, options.loglevel) # we use an Autobahn utility to import the "best" available Twisted reactor # from autobahn.twisted.choosereactor import install_reactor reactor = install_reactor(options.reactor) WORKER_TYPE_TO_TITLE = { 'router': 'Router', 'container': 'Container', 'websocket-testee': 'WebSocket Testee' } from twisted.python.reflect import qual log.info( '{worker_title} worker "{worker_id}" process {pid} starting on {python}-{reactor} ..', worker_title=WORKER_TYPE_TO_TITLE[options.type], worker_id=options.worker, pid=os.getpid(), python=platform.python_implementation(), reactor=qual(reactor.__class__).split('.')[-1]) # set process title if requested to # try: import setproctitle except ImportError: log.debug( "Could not set worker process title (setproctitle not installed)") else: if options.title: setproctitle.setproctitle(options.title) else: WORKER_TYPE_TO_PROCESS_TITLE = { 'router': 'crossbar-worker [router]', 'container': 'crossbar-worker [container]', 'websocket-testee': 'crossbar-worker [websocket-testee]' } setproctitle.setproctitle( WORKER_TYPE_TO_PROCESS_TITLE[options.type].strip()) # node directory # options.cbdir = os.path.abspath(options.cbdir) os.chdir(options.cbdir) # log.msg("Starting from node directory {}".format(options.cbdir)) from crossbar.worker.router import RouterWorkerSession from crossbar.worker.container import ContainerWorkerSession from crossbar.worker.testee import WebSocketTesteeWorkerSession WORKER_TYPE_TO_CLASS = { 'router': RouterWorkerSession, 'container': ContainerWorkerSession, 'websocket-testee': WebSocketTesteeWorkerSession } from twisted.internet.error import ConnectionDone from autobahn.twisted.websocket import WampWebSocketServerProtocol class WorkerServerProtocol(WampWebSocketServerProtocol): def connectionLost(self, reason): # the behavior here differs slightly whether we're shutting down orderly # or shutting down because of "issues" if isinstance(reason.value, ConnectionDone): was_clean = True else: was_clean = False try: # this log message is unlikely to reach the controller (unless # only stdin/stdout pipes were lost, but not stderr) if was_clean: log.info("Connection to node controller closed cleanly") else: log.warn("Connection to node controller lost: {reason}", reason=reason) # give the WAMP transport a change to do it's thing WampWebSocketServerProtocol.connectionLost(self, reason) except: # we're in the process of shutting down .. so ignore .. pass finally: # after the connection to the node controller is gone, # the worker is "orphane", and should exit # determine process exit code if was_clean: exit_code = 0 else: exit_code = 1 # exit the whole worker process when the reactor has stopped reactor.addSystemEventTrigger('after', 'shutdown', os._exit, exit_code) # stop the reactor try: reactor.stop() except ReactorNotRunning: pass try: # create a WAMP application session factory # from autobahn.twisted.wamp import ApplicationSessionFactory from autobahn.wamp.types import ComponentConfig session_config = ComponentConfig(realm=options.realm, extra=options) session_factory = ApplicationSessionFactory(session_config) session_factory.session = WORKER_TYPE_TO_CLASS[options.type] # create a WAMP-over-WebSocket transport server factory # from autobahn.twisted.websocket import WampWebSocketServerFactory transport_factory = WampWebSocketServerFactory(session_factory, u'ws://localhost') transport_factory.protocol = WorkerServerProtocol transport_factory.setProtocolOptions(failByDrop=False) # create a protocol instance and wire up to stdio # from twisted.python.runtime import platform as _platform from twisted.internet import stdio proto = transport_factory.buildProtocol(None) if _platform.isWindows(): stdio.StandardIO(proto) else: stdio.StandardIO(proto, stdout=3) # now start reactor loop # if False: log.info("vmprof enabled.") import os import vmprof PROFILE_FILE = 'vmprof_{}.dat'.format(os.getpid()) outfd = os.open(PROFILE_FILE, os.O_RDWR | os.O_CREAT | os.O_TRUNC) vmprof.enable(outfd, period=0.01) log.info("Entering event loop...") reactor.run() vmprof.disable() else: log.debug("Entering event loop...") reactor.run() except Exception as e: log.info("Unhandled exception: {e}", e=e) if reactor.running: reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1) reactor.stop() else: sys.exit(1)