def onConnect(self, do_join=True): """ """ if not hasattr(self, 'cbdir'): self.cbdir = self.config.extra.cbdir if not hasattr(self, '_uri_prefix'): self._uri_prefix = 'crossbar.node.{}'.format( self.config.extra.node) self._started = datetime.utcnow() # see: BaseSession self.include_traceback = False self._manhole_service = None if _HAS_PSUTIL: self._pinfo = ProcessInfo() self._pinfo_monitor = None self._pinfo_monitor_seq = 0 else: self._pinfo = None self._pinfo_monitor = None self._pinfo_monitor_seq = None self.log.info("Process utilities not available") self._connections = {} if do_join: self.join(self.config.realm)
def onJoin(self, details): self._db = Database(dbpath='../results') self._schema = Schema.attach(self._db) self._pinfo = ProcessInfo() self._logname = self.config.extra['logname'] self._period = self.config.extra.get('period', 10.) self._running = True batch_id = uuid.uuid4() self._last_stats = None self._stats_loop = LoopingCall(self._stats, batch_id) self._stats_loop.start(self._period) dl = [] for i in range(8): d = self._loop(batch_id, i) dl.append(d) d = gatherResults(dl) try: yield d except TransportLost: pass
def onJoin(self, details): self._prefix = self.config.extra['prefix'] self._logname = self.config.extra['logname'] self._pinfo = ProcessInfo() def echo(arg): return arg yield self.register(echo, '{}.echo'.format(self._prefix), options=RegisterOptions(invoke='roundrobin')) self.log.info('{} ready!'.format(self._logname)) last = None while True: stats = self._pinfo.get_stats() if last: secs = (stats['time'] - last['time']) / 10**9 ctx = round((stats['voluntary'] - last['voluntary']) / secs, 0) self.log.info('{logname}: {cpu} cpu, {mem} mem, {ctx} ctx', logname=self._logname, cpu=round(stats['cpu_percent'], 1), mem=round(stats['mem_percent'], 1), ctx=ctx) last = stats yield sleep(5)
class AppSession(ApplicationSession): log = txaio.make_logger() @inlineCallbacks def onJoin(self, details): self._prefix = self.config.extra['prefix'] self._logname = self.config.extra['logname'] self._pinfo = ProcessInfo() def echo(arg): return arg yield self.register(echo, '{}.echo'.format(self._prefix), options=RegisterOptions(invoke='roundrobin')) self.log.info('{} ready!'.format(self._logname)) last = None while True: stats = self._pinfo.get_stats() if last: secs = (stats['time'] - last['time']) / 10**9 ctx = round((stats['voluntary'] - last['voluntary']) / secs, 0) self.log.info('{logname}: {cpu} cpu, {mem} mem, {ctx} ctx', logname=self._logname, cpu=round(stats['cpu_percent'], 1), mem=round(stats['mem_percent'], 1), ctx=ctx) last = stats yield sleep(5)
def onConnect(self, do_join=True): """ """ if not hasattr(self, 'cbdir'): self.cbdir = self.config.extra.cbdir if not hasattr(self, '_uri_prefix'): self._uri_prefix = 'crossbar.node.{}'.format(self.config.extra.node) self._started = datetime.utcnow() # see: BaseSession self.include_traceback = False self.debug_app = False self._manhole_service = None if _HAS_PSUTIL: self._pinfo = ProcessInfo() self._pinfo_monitor = None self._pinfo_monitor_seq = 0 else: self._pinfo = None self._pinfo_monitor = None self._pinfo_monitor_seq = None self.log.info("Process utilities not available") self._connections = {} if do_join: self.join(self.config.realm)
def onJoin(self, details): self._prefix = self.config.extra['prefix'] self._logname = self.config.extra['logname'] self._count = 0 self._pinfo = ProcessInfo() def echo(arg): self._count += 1 return arg yield self.register(echo, '{}.echo'.format(self._prefix), options=RegisterOptions(invoke='roundrobin')) self.log.info('{} ready!'.format(self._logname)) last = None while True: stats = self._pinfo.get_stats() if last: secs = (stats['time'] - last['time']) / 10**9 calls_per_sec = int(round(self._count / secs, 0)) ctx = round((stats['voluntary'] - last['voluntary']) / secs, 0) self.log.info( '{logprefix}: {user} user, {system} system, {mem_percent} mem_percent, {ctx} ctx', logprefix=hl('LOAD {}'.format(self._logname), color='cyan', bold=True), user=stats['user'], system=stats['system'], mem_percent=round(stats['mem_percent'], 1), ctx=ctx) self.log.info( '{logprefix}: {calls} calls, {calls_per_sec} calls/second', logprefix=hl('WAMP {}'.format(self._logname), color='cyan', bold=True), calls=self._count, calls_per_sec=hl(calls_per_sec, bold=True)) self._count = 0 last = stats yield sleep(10)
def onJoin(self, details): self._db = Database(dbpath='../results') self._schema = Schema.attach(self._db) self._pinfo = ProcessInfo() self._logname = self.config.extra['logname'] self._period = self.config.extra.get('period', 5.) self._running = True dl = [] for i in range(8): d = self._loop(i) dl.append(d) d = gatherResults(dl) try: yield d except TransportLost: pass
def onConnect(self, do_join=True): """ """ if not hasattr(self, 'debug'): self.debug = self.config.extra.debug if not hasattr(self, 'cbdir'): self.cbdir = self.config.extra.cbdir if not hasattr(self, '_uri_prefix'): self._uri_prefix = 'crossbar.node.{}'.format( self.config.extra.node) if self.debug: log.msg("Session connected to management router") self._started = datetime.utcnow() # see: BaseSession self.include_traceback = False self.debug_app = False self._manhole_service = None if _HAS_PSUTIL: self._pinfo = ProcessInfo() self._pinfo_monitor = None self._pinfo_monitor_seq = 0 else: self._pinfo = None self._pinfo_monitor = None self._pinfo_monitor_seq = None log.msg("Warning: process utilities not available") if do_join: self.join(self.config.realm)
def on_worker_connected(self, proto): """ Called immediately after the worker process has been forked. IMPORTANT: this slightly differs between native workers and guest workers! """ assert (self.status == u'starting') assert (self.connected is None) assert (self.proto is None) assert (self.pid is None) assert (self.pinfo is None) self.status = u'connected' self.connected = datetime.utcnow() self.proto = proto self.pid = proto.transport.pid self.pinfo = ProcessInfo(self.pid)
def on_worker_started(self, proto=None): """ Called after the worker process is connected to the node router and registered all its management APIs there. The worker is now ready for use! """ assert (self.status in [u'starting', u'connected']) assert (self.started is None) assert (self.proto is not None or proto is not None) if not self.pid: self.pid = proto.transport.pid if not self.pinfo: self.pinfo = ProcessInfo(self.pid) assert (self.pid is not None) assert (self.pinfo is not None) self.status = u'started' self.proto = self.proto or proto self.started = datetime.utcnow()
class NativeProcessSession(ApplicationSession): """ A native Crossbar.io process (currently: controller, router or container). """ log = make_logger() def __init__(self, config=None, reactor=None): if not reactor: from twisted.internet import reactor self._reactor = reactor self._reactor = reactor super(ApplicationSession, self).__init__(config=config) def onConnect(self, do_join=True): """ """ if not hasattr(self, 'cbdir'): self.cbdir = self.config.extra.cbdir if not hasattr(self, '_uri_prefix'): self._uri_prefix = 'crossbar.node.{}'.format( self.config.extra.node) self._started = datetime.utcnow() # see: BaseSession self.include_traceback = False self._manhole_service = None if _HAS_PSUTIL: self._pinfo = ProcessInfo() self._pinfo_monitor = None self._pinfo_monitor_seq = 0 else: self._pinfo = None self._pinfo_monitor = None self._pinfo_monitor_seq = None self.log.info("Process utilities not available") self._connections = {} if do_join: self.join(self.config.realm) @inlineCallbacks def onJoin(self, details): """ Called when process has joined the node's management realm. """ regs = yield self.register( self, prefix=u'{}.'.format(self._uri_prefix), options=RegisterOptions(details_arg='details'), ) self.log.info("Registered {len_reg} procedures", len_reg=len(regs)) for reg in regs: if isinstance(reg, Failure): self.log.error("Failed to register: {f}", f=reg, log_failure=reg) else: self.log.info(' {proc}', proc=reg.procedure) returnValue(regs) @inlineCallbacks def start_connection(self, id, config, details=None): """ Starts a connection in this process. :param id: The ID for the started connection. :type id: unicode :param config: Connection configuration. :type config: dict :param details: Caller details. :type details: instance of :class:`autobahn.wamp.types.CallDetails` :returns dict -- The connection. """ self.log.debug("start_connection: id={id}, config={config}", id=id, config=config) # prohibit starting a component twice # if id in self._connections: emsg = "cannot start connection: a connection with id={} is already started".format( id) self.log.warn(emsg) raise ApplicationError(u"crossbar.error.invalid_configuration", emsg) # check configuration # try: checkconfig.check_connection(config) except Exception as e: emsg = "invalid connection configuration ({})".format(e) self.log.warn(emsg) raise ApplicationError(u"crossbar.error.invalid_configuration", emsg) else: self.log.info("Starting {ptype} in process.", ptype=config['type']) if config['type'] == u'postgresql.connection': if _HAS_POSTGRESQL: connection = PostgreSQLConnection(id, config) else: emsg = "unable to start connection - required PostgreSQL driver package not installed" self.log.warn(emsg) raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) else: # should not arrive here raise Exception("logic error") self._connections[id] = connection try: yield connection.start() self.log.info( "Connection {connection_type} started '{connection_id}'", connection_id=id, connection_type=config['type']) except Exception as e: del self._connections[id] raise state = connection.marshal() self.publish(u'crossbar.node.process.on_connection_start', state) returnValue(state) @inlineCallbacks def stop_connection(self, id, details=None): """ Stop a connection currently running within this process. :param id: The ID of the connection to stop. :type id: unicode :param details: Caller details. :type details: instance of :class:`autobahn.wamp.types.CallDetails` :returns dict -- A dict with component start information. """ self.log.debug("stop_connection: id={id}", id=id) if id not in self._connections: raise ApplicationError( u'crossbar.error.no_such_object', 'no connection with ID {} running in this process'.format(id)) connection = self._connections[id] try: yield connection.stop() except Exception as e: self.log.warn('could not stop connection {id}: {error}', error=e) raise del self._connections[id] state = connection.marshal() self.publish(u'crossbar.node.process.on_connection_stop', state) returnValue(state) def get_connections(self, details=None): """ Get connections currently running within this processs. :param details: Caller details. :type details: instance of :class:`autobahn.wamp.types.CallDetails` :returns list -- List of connections. """ self.log.debug("get_connections") res = [] for c in self._connections.values(): res.append(c.marshal()) return res @wamp.register(None) def get_process_info(self, details=None): """ Get process information (open files, sockets, ...). :returns: dict -- Dictionary with process information. """ self.log.debug("{cls}.get_process_info", cls=self.__class__.__name__) if self._pinfo: return self._pinfo.get_info() else: emsg = "Could not retrieve process statistics: required packages not installed" raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) @wamp.register(None) def get_process_stats(self, details=None): """ Get process statistics (CPU, memory, I/O). :returns: dict -- Dictionary with process statistics. """ self.log.debug("{cls}.get_process_stats", cls=self.__class__.__name__) if self._pinfo: return self._pinfo.get_stats() else: emsg = "Could not retrieve process statistics: required packages not installed" raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) @wamp.register(None) def set_process_stats_monitoring(self, interval, details=None): """ Enable/disable periodic publication of process statistics. :param interval: The monitoring interval in seconds. Set to 0 to disable monitoring. :type interval: float """ self.log.debug( "{cls}.set_process_stats_monitoring(interval = {interval})", cls=self.__class__.__name__, interval=interval) if self._pinfo: stats_monitor_set_topic = '{}.on_process_stats_monitoring_set'.format( self._uri_prefix) # stop and remove any existing monitor if self._pinfo_monitor: self._pinfo_monitor.stop() self._pinfo_monitor = None self.publish(stats_monitor_set_topic, 0, options=PublishOptions(exclude=details.caller)) # possibly start a new monitor if interval > 0: stats_topic = '{}.on_process_stats'.format(self._uri_prefix) def publish_stats(): stats = self._pinfo.get_stats() self._pinfo_monitor_seq += 1 stats[u'seq'] = self._pinfo_monitor_seq self.publish(stats_topic, stats) self._pinfo_monitor = LoopingCall(publish_stats) self._pinfo_monitor.start(interval) self.publish(stats_monitor_set_topic, interval, options=PublishOptions(exclude=details.caller)) else: emsg = "Cannot setup process statistics monitor: required packages not installed" raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) @wamp.register(None) def trigger_gc(self, details=None): """ Manually trigger a garbage collection in this native process. This procedure is registered under ``crossbar.node.<node_id>.worker.<worker_id>.trigger_gc`` for native workers and under ``crossbar.node.<node_id>.controller.trigger_gc`` for node controllers. The procedure will publish an event when the garabage collection has finished to ``crossbar.node.<node_id>.worker.<worker_id>.on_gc_finished`` for native workers and ``crossbar.node.<node_id>.controller.on_gc_finished`` for node controllers: .. code-block:: javascript { "requester": { "session_id": 982734923, "auth_id": "bob", "auth_role": "admin" }, "duration": 190 } .. note:: The caller of this procedure will NOT receive the event. :returns: Time (wall clock) consumed for garbage collection in ms. :rtype: int """ self.log.debug("{cls}.trigger_gc", cls=self.__class__.__name__) started = rtime() # now trigger GC .. this is blocking! gc.collect() duration = int(round(1000. * (rtime() - started))) on_gc_finished = u'{}.on_gc_finished'.format(self._uri_prefix) self.publish( on_gc_finished, { u'requester': { u'session_id': details.caller, # FIXME: u'auth_id': None, u'auth_role': None }, u'duration': duration }, options=PublishOptions(exclude=details.caller)) return duration @wamp.register(None) @inlineCallbacks def start_manhole(self, config, details=None): """ Start a Manhole service within this process. **Usage:** This procedure is registered under * ``crossbar.node.<node_id>.worker.<worker_id>.start_manhole`` - for native workers * ``crossbar.node.<node_id>.controller.start_manhole`` - for node controllers The procedure takes a Manhole service configuration which defines a listening endpoint for the service and a list of users including passwords, e.g. .. code-block:: javascript { "endpoint": { "type": "tcp", "port": 6022 }, "users": [ { "user": "******", "password": "******" } ] } **Errors:** The procedure may raise the following errors: * ``crossbar.error.invalid_configuration`` - the provided configuration is invalid * ``crossbar.error.already_started`` - the Manhole service is already running (or starting) * ``crossbar.error.feature_unavailable`` - the required support packages are not installed **Events:** The procedure will publish an event when the service **is starting** to * ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_starting`` - for native workers * ``crossbar.node.<node_id>.controller.on_manhole_starting`` - for node controllers and publish an event when the service **has started** to * ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_started`` - for native workers * ``crossbar.node.<node_id>.controller.on_manhole_started`` - for node controllers :param config: Manhole service configuration. :type config: dict """ self.log.debug("{cls}.start_manhole(config = {config})", cls=self.__class__.__name__, config=config) if not _HAS_MANHOLE: emsg = "Could not start manhole: required packages are missing ({})".format( _MANHOLE_MISSING_REASON) self.log.error(emsg) raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) if self._manhole_service: emsg = "Could not start manhole - already running (or starting)" self.log.warn(emsg) raise ApplicationError(u"crossbar.error.already_started", emsg) try: checkconfig.check_manhole(config) except Exception as e: emsg = "Could not start manhole: invalid configuration ({})".format( e) self.log.error(emsg) raise ApplicationError(u'crossbar.error.invalid_configuration', emsg) # setup user authentication # checker = checkers.InMemoryUsernamePasswordDatabaseDontUse() for user in config['users']: checker.addUser(user['user'], user['password']) # setup manhole namespace # namespace = {'session': self} from twisted.conch.manhole_ssh import (ConchFactory, TerminalRealm, TerminalSession) from twisted.conch.manhole import ColoredManhole class PatchedTerminalSession(TerminalSession): # get rid of # exceptions.AttributeError: TerminalSession instance has no attribute 'windowChanged' def windowChanged(self, winSize): pass rlm = TerminalRealm() rlm.sessionFactory = PatchedTerminalSession # monkey patch rlm.chainedProtocolFactory.protocolFactory = lambda _: ColoredManhole( namespace) ptl = portal.Portal(rlm, [checker]) factory = ConchFactory(ptl) factory.noisy = False self._manhole_service = ManholeService(config, details.caller) starting_topic = '{}.on_manhole_starting'.format(self._uri_prefix) starting_info = self._manhole_service.marshal() # the caller gets a progressive result .. if details.progress: details.progress(starting_info) # .. while all others get an event self.publish(starting_topic, starting_info, options=PublishOptions(exclude=details.caller)) try: self._manhole_service.port = yield create_listening_port_from_config( config['endpoint'], self.cbdir, factory, self._reactor, self.log) except Exception as e: self._manhole_service = None emsg = "Manhole service endpoint cannot listen: {}".format(e) self.log.error(emsg) raise ApplicationError(u"crossbar.error.cannot_listen", emsg) # alright, manhole has started self._manhole_service.started = datetime.utcnow() self._manhole_service.status = 'started' started_topic = '{}.on_manhole_started'.format(self._uri_prefix) started_info = self._manhole_service.marshal() self.publish(started_topic, started_info, options=PublishOptions(exclude=details.caller)) returnValue(started_info) @wamp.register(None) @inlineCallbacks def stop_manhole(self, details=None): """ Stop the Manhole service running in this process. This procedure is registered under * ``crossbar.node.<node_id>.worker.<worker_id>.stop_manhole`` for native workers and under * ``crossbar.node.<node_id>.controller.stop_manhole`` for node controllers When no Manhole service is currently running within this process, or the Manhole service is already shutting down, a ``crossbar.error.not_started`` WAMP error is raised. The procedure will publish an event when the service **is stopping** to * ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_stopping`` for native workers and * ``crossbar.node.<node_id>.controller.on_manhole_stopping`` for node controllers and will publish an event when the service **has stopped** to * ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_stopped`` for native workers and * ``crossbar.node.<node_id>.controller.on_manhole_stopped`` for node controllers """ self.log.debug("{cls}.stop_manhole", cls=self.__class__.__name__) if not self._manhole_service or self._manhole_service.status != 'started': emsg = "Cannot stop manhole: not running (or already shutting down)" raise ApplicationError(u"crossbar.error.not_started", emsg) self._manhole_service.status = 'stopping' stopping_topic = u'{}.on_manhole_stopping'.format(self._uri_prefix) stopping_info = None # the caller gets a progressive result .. if details.progress: details.progress(stopping_info) # .. while all others get an event self.publish(stopping_topic, stopping_info, options=PublishOptions(exclude=details.caller)) try: yield self._manhole_service.port.stopListening() except Exception as e: self.log.warn("error while stop listening on endpoint: {error}", error=e) self._manhole_service = None stopped_topic = u'{}.on_manhole_stopped'.format(self._uri_prefix) stopped_info = None self.publish(stopped_topic, stopped_info, options=PublishOptions(exclude=details.caller)) returnValue(stopped_info) @wamp.register(None) def get_manhole(self, details=None): """ Get current manhole service information. :returns: dict -- A dict with service information or `None` if the service is not running. """ self.log.debug("{cls}.get_manhole", cls=self.__class__.__name__) if not _HAS_MANHOLE: emsg = "Could not start manhole: required packages are missing ({})".format( _MANHOLE_MISSING_REASON) self.log.error(emsg) raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) if not self._manhole_service: return None else: return self._manhole_service.marshal() @wamp.register(None) def utcnow(self, details=None): """ Return current time as determined from within this process. **Usage:** This procedure is registered under * ``crossbar.node.<node_id>.worker.<worker_id>.utcnow`` for native workers and under * ``crossbar.node.<node_id>.controller.utcnow`` for node controllers :returns: Current time (UTC) in UTC ISO 8601 format. :rtype: unicode """ self.log.debug("{cls}.utcnow", cls=self.__class__.__name__) return utcnow() @wamp.register(None) def started(self, details=None): """ Return start time of this process. **Usage:** This procedure is registered under * ``crossbar.node.<node_id>.worker.<worker_id>.started`` for native workers and under * ``crossbar.node.<node_id>.controller.started`` for node controllers :returns: Start time (UTC) in UTC ISO 8601 format. :rtype: unicode """ self.log.debug("{cls}.started", cls=self.__class__.__name__) return utcstr(self._started) @wamp.register(None) def uptime(self, details=None): """ Return uptime of this process. **Usage:** This procedure is registered under * ``crossbar.node.<node_id>.worker.<worker_id>.uptime`` for native workers and under * ``crossbar.node.<node_id>.controller.uptime`` for node controllers :returns: Uptime in seconds. :rtype: float """ self.log.debug("{cls}.uptime", cls=self.__class__.__name__) now = datetime.utcnow() return (now - self._started).total_seconds()
class NativeProcessSession(ApplicationSession): """ A native Crossbar.io process (currently: controller, router or container). """ log = make_logger() def __init__(self, config=None, reactor=None): if not reactor: from twisted.internet import reactor self._reactor = reactor self._reactor = reactor super(ApplicationSession, self).__init__(config=config) def onConnect(self, do_join=True): """ """ if not hasattr(self, 'cbdir'): self.cbdir = self.config.extra.cbdir if not hasattr(self, '_uri_prefix'): self._uri_prefix = 'crossbar.node.{}'.format( self.config.extra.node) self._started = datetime.utcnow() # see: BaseSession self.include_traceback = False self.debug_app = False self._manhole_service = None if _HAS_PSUTIL: self._pinfo = ProcessInfo() self._pinfo_monitor = None self._pinfo_monitor_seq = 0 else: self._pinfo = None self._pinfo_monitor = None self._pinfo_monitor_seq = None self.log.info("Process utilities not available") self._connections = {} if do_join: self.join(self.config.realm) @inlineCallbacks def onJoin(self, details): """ Called when process has joined the node's management realm. """ procs = [ 'start_manhole', 'stop_manhole', 'get_manhole', 'start_connection', 'stop_connection', 'get_connections', 'trigger_gc', 'utcnow', 'started', 'uptime', 'get_process_info', 'get_process_stats', 'set_process_stats_monitoring' ] dl = [] for proc in procs: uri = '{}.{}'.format(self._uri_prefix, proc) self.log.debug("Registering procedure '{uri}'", uri=uri) dl.append( self.register(getattr(self, proc), uri, options=RegisterOptions(details_arg='details'))) regs = yield DeferredList(dl) self.log.debug("Registered {len_reg} procedures", len_reg=len(regs)) @inlineCallbacks def start_connection(self, id, config, details=None): """ Starts a connection in this process. :param id: The ID for the started connection. :type id: unicode :param config: Connection configuration. :type config: dict :param details: Caller details. :type details: instance of :class:`autobahn.wamp.types.CallDetails` :returns dict -- The connection. """ self.log.debug("start_connection: id={id}, config={config}", id=id, config=config) # prohibit starting a component twice # if id in self._connections: emsg = "cannot start connection: a connection with id={} is already started".format( id) self.log.warn(emsg) raise ApplicationError("crossbar.error.invalid_configuration", emsg) # check configuration # try: checkconfig.check_connection(config) except Exception as e: emsg = "invalid connection configuration ({})".format(e) self.log.warn(emsg) raise ApplicationError("crossbar.error.invalid_configuration", emsg) else: self.log.info("Starting {} in process.".format(config['type'])) if config['type'] == u'postgresql.connection': if _HAS_POSTGRESQL: connection = PostgreSQLConnection(id, config) else: emsg = "unable to start connection - required PostgreSQL driver package not installed" self.log.warn(emsg) raise ApplicationError("crossbar.error.feature_unavailable", emsg) else: # should not arrive here raise Exception("logic error") self._connections[id] = connection try: yield connection.start() self.log.info( "Connection {connection_type} started '{connection_id}'", connection_id=id, connection_type=config['type']) except Exception as e: del self._connections[id] raise state = connection.marshal() self.publish(u'crossbar.node.process.on_connection_start', state) returnValue(state) @inlineCallbacks def stop_connection(self, id, details=None): """ Stop a connection currently running within this process. :param id: The ID of the connection to stop. :type id: unicode :param details: Caller details. :type details: instance of :class:`autobahn.wamp.types.CallDetails` :returns dict -- A dict with component start information. """ self.log.debug("stop_connection: id={id}", id=id) if id not in self._connections: raise ApplicationError( 'crossbar.error.no_such_object', 'no connection with ID {} running in this process'.format(id)) connection = self._connections[id] try: yield connection.stop() except Exception as e: self.log.warn('could not stop connection {id}: {error}', error=e) raise del self._connections[id] state = connection.marshal() self.publish(u'crossbar.node.process.on_connection_stop', state) returnValue(state) def get_connections(self, details=None): """ Get connections currently running within this processs. :param details: Caller details. :type details: instance of :class:`autobahn.wamp.types.CallDetails` :returns list -- List of connections. """ self.log.debug("get_connections") res = [] for c in self._connections.values(): res.append(c.marshal()) return res def get_process_info(self, details=None): """ Get process information (open files, sockets, ...). :returns: dict -- Dictionary with process information. """ self.log.debug("{cls}.get_process_info", cls=self.__class__.__name__) if self._pinfo: return self._pinfo.get_info() else: emsg = "ERROR: could not retrieve process statistics - required packages not installed" raise ApplicationError("crossbar.error.feature_unavailable", emsg) def get_process_stats(self, details=None): """ Get process statistics (CPU, memory, I/O). :returns: dict -- Dictionary with process statistics. """ self.log.debug("{cls}.get_process_stats", cls=self.__class__.__name__) if self._pinfo: return self._pinfo.get_stats() else: emsg = "ERROR: could not retrieve process statistics - required packages not installed" raise ApplicationError("crossbar.error.feature_unavailable", emsg) def set_process_stats_monitoring(self, interval, details=None): """ Enable/disable periodic publication of process statistics. :param interval: The monitoring interval in seconds. Set to 0 to disable monitoring. :type interval: float """ self.log.debug( "{cls}.set_process_stats_monitoring(interval = {interval})", cls=self.__class__.__name__, interval=interval) if self._pinfo: stats_monitor_set_topic = '{}.on_process_stats_monitoring_set'.format( self._uri_prefix) # stop and remove any existing monitor if self._pinfo_monitor: self._pinfo_monitor.stop() self._pinfo_monitor = None self.publish(stats_monitor_set_topic, 0, options=PublishOptions(exclude=[details.caller])) # possibly start a new monitor if interval > 0: stats_topic = '{}.on_process_stats'.format(self._uri_prefix) def publish_stats(): stats = self._pinfo.get_stats() self._pinfo_monitor_seq += 1 stats['seq'] = self._pinfo_monitor_seq self.publish(stats_topic, stats) self._pinfo_monitor = LoopingCall(publish_stats) self._pinfo_monitor.start(interval) self.publish(stats_monitor_set_topic, interval, options=PublishOptions(exclude=[details.caller])) else: emsg = "ERROR: cannot setup process statistics monitor - required packages not installed" raise ApplicationError("crossbar.error.feature_unavailable", emsg) def trigger_gc(self, details=None): """ Triggers a garbage collection. :returns: float -- Time consumed for GC in ms. """ self.msg.debug("{cls}.trigger_gc", cls=self.__class__.__name__) started = rtime() gc.collect() return 1000. * (rtime() - started) @inlineCallbacks def start_manhole(self, config, details=None): """ Start a manhole (SSH) within this worker. :param config: Manhole configuration. :type config: obj """ self.log.debug("{cls}.start_manhole(config = {config})", cls=self.__class__.__name__, config=config) if not _HAS_MANHOLE: emsg = "ERROR: could not start manhole - required packages are missing ({})".format( _MANHOLE_MISSING_REASON) self.log.error(emsg) raise ApplicationError("crossbar.error.feature_unavailable", emsg) if self._manhole_service: emsg = "ERROR: could not start manhole - already running (or starting)" self.log.warn(emsg) raise ApplicationError("crossbar.error.already_started", emsg) try: checkconfig.check_manhole(config) except Exception as e: emsg = "ERROR: could not start manhole - invalid configuration ({})".format( e) self.log.error(emsg) raise ApplicationError('crossbar.error.invalid_configuration', emsg) # setup user authentication # checker = checkers.InMemoryUsernamePasswordDatabaseDontUse() for user in config['users']: checker.addUser(user['user'], user['password']) # setup manhole namespace # namespace = {'session': self} class PatchedTerminalSession(TerminalSession): # get rid of # exceptions.AttributeError: TerminalSession instance has no attribute 'windowChanged' def windowChanged(self, winSize): pass rlm = TerminalRealm() rlm.sessionFactory = PatchedTerminalSession # monkey patch rlm.chainedProtocolFactory.protocolFactory = lambda _: ColoredManhole( namespace) ptl = portal.Portal(rlm, [checker]) factory = ConchFactory(ptl) factory.noisy = False self._manhole_service = ManholeService(config, details.caller) starting_topic = '{}.on_manhole_starting'.format(self._uri_prefix) starting_info = self._manhole_service.marshal() # the caller gets a progressive result .. if details.progress: details.progress(starting_info) # .. while all others get an event self.publish(starting_topic, starting_info, options=PublishOptions(exclude=[details.caller])) try: self._manhole_service.port = yield create_listening_port_from_config( config['endpoint'], factory, self.cbdir, self._reactor) except Exception as e: self._manhole_service = None emsg = "ERROR: manhole service endpoint cannot listen - {}".format( e) self.log.error(emsg) raise ApplicationError("crossbar.error.cannot_listen", emsg) # alright, manhole has started self._manhole_service.started = datetime.utcnow() self._manhole_service.status = 'started' started_topic = '{}.on_manhole_started'.format(self._uri_prefix) started_info = self._manhole_service.marshal() self.publish(started_topic, started_info, options=PublishOptions(exclude=[details.caller])) returnValue(started_info) @inlineCallbacks def stop_manhole(self, details=None): """ Stop Manhole. """ self.log.debug("{cls}.stop_manhole", cls=self.__class__.__name__) if not _HAS_MANHOLE: emsg = "ERROR: could not start manhole - required packages are missing ({})".format( _MANHOLE_MISSING_REASON) self.log.error(emsg) raise ApplicationError("crossbar.error.feature_unavailable", emsg) if not self._manhole_service or self._manhole_service.status != 'started': emsg = "ERROR: cannot stop manhole - not running (or already shutting down)" raise ApplicationError("crossbar.error.not_started", emsg) self._manhole_service.status = 'stopping' stopping_topic = '{}.on_manhole_stopping'.format(self._uri_prefix) stopping_info = None # the caller gets a progressive result .. if details.progress: details.progress(stopping_info) # .. while all others get an event self.publish(stopping_topic, stopping_info, options=PublishOptions(exclude=[details.caller])) try: yield self._manhole_service.port.stopListening() except Exception as e: raise Exception( "INTERNAL ERROR: don't know how to handle a failed called to stopListening() - {}" .format(e)) self._manhole_service = None stopped_topic = '{}.on_manhole_stopped'.format(self._uri_prefix) stopped_info = None self.publish(stopped_topic, stopped_info, options=PublishOptions(exclude=[details.caller])) returnValue(stopped_info) def get_manhole(self, details=None): """ Get current manhole service information. :returns: dict -- A dict with service information or `None` if the service is not running. """ self.log.debug("{cls}.get_manhole", cls=self.__class__.__name__) if not _HAS_MANHOLE: emsg = "ERROR: could not start manhole - required packages are missing ({})".format( _MANHOLE_MISSING_REASON) self.log.error(emsg) raise ApplicationError("crossbar.error.feature_unavailable", emsg) if not self._manhole_service: return None else: return self._manhole_service.marshal() def utcnow(self, details=None): """ Return current time as determined from within this process. :returns str -- Current time (UTC) in UTC ISO 8601 format. """ self.log.debug("{cls}.utcnow", cls=self.__class__.__name__) return utcnow() def started(self, details=None): """ Return start time of this process. :returns str -- Start time (UTC) in UTC ISO 8601 format. """ self.log.debug("{cls}.started", cls=self.__class__.__name__) return utcstr(self._started) def uptime(self, details=None): """ Uptime of this process. :returns float -- Uptime in seconds. """ self.log.debug("{cls}.uptime", cls=self.__class__.__name__) now = datetime.utcnow() return (now - self._started).total_seconds()
class NativeProcessSession(ApplicationSession): """ A native Crossbar.io process (currently: controller, router or container). """ log = make_logger() def __init__(self, config=None, reactor=None): if not reactor: from twisted.internet import reactor self._reactor = reactor self._reactor = reactor super(ApplicationSession, self).__init__(config=config) def onConnect(self, do_join=True): """ """ if not hasattr(self, 'cbdir'): self.cbdir = self.config.extra.cbdir if not hasattr(self, '_uri_prefix'): self._uri_prefix = 'crossbar.node.{}'.format(self.config.extra.node) self._started = datetime.utcnow() # see: BaseSession self.include_traceback = False self.debug_app = False self._manhole_service = None if _HAS_PSUTIL: self._pinfo = ProcessInfo() self._pinfo_monitor = None self._pinfo_monitor_seq = 0 else: self._pinfo = None self._pinfo_monitor = None self._pinfo_monitor_seq = None self.log.info("Process utilities not available") self._connections = {} if do_join: self.join(self.config.realm) @inlineCallbacks def onJoin(self, details): """ Called when process has joined the node's management realm. """ procs = [ 'start_manhole', 'stop_manhole', 'get_manhole', 'start_connection', 'stop_connection', 'get_connections', 'trigger_gc', 'utcnow', 'started', 'uptime', 'get_process_info', 'get_process_stats', 'set_process_stats_monitoring' ] dl = [] for proc in procs: uri = '{}.{}'.format(self._uri_prefix, proc) self.log.debug("Registering procedure '{uri}'", uri=uri) dl.append(self.register(getattr(self, proc), uri, options=RegisterOptions(details_arg='details'))) regs = yield DeferredList(dl) self.log.debug("Registered {len_reg} procedures", len_reg=len(regs)) @inlineCallbacks def start_connection(self, id, config, details=None): """ Starts a connection in this process. :param id: The ID for the started connection. :type id: unicode :param config: Connection configuration. :type config: dict :param details: Caller details. :type details: instance of :class:`autobahn.wamp.types.CallDetails` :returns dict -- The connection. """ self.log.debug("start_connection: id={id}, config={config}", id=id, config=config) # prohibit starting a component twice # if id in self._connections: emsg = "cannot start connection: a connection with id={} is already started".format(id) self.log.warn(emsg) raise ApplicationError(u"crossbar.error.invalid_configuration", emsg) # check configuration # try: checkconfig.check_connection(config) except Exception as e: emsg = "invalid connection configuration ({})".format(e) self.log.warn(emsg) raise ApplicationError(u"crossbar.error.invalid_configuration", emsg) else: self.log.info("Starting {} in process.".format(config['type'])) if config['type'] == u'postgresql.connection': if _HAS_POSTGRESQL: connection = PostgreSQLConnection(id, config) else: emsg = "unable to start connection - required PostgreSQL driver package not installed" self.log.warn(emsg) raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) else: # should not arrive here raise Exception("logic error") self._connections[id] = connection try: yield connection.start() self.log.info("Connection {connection_type} started '{connection_id}'", connection_id=id, connection_type=config['type']) except Exception as e: del self._connections[id] raise state = connection.marshal() self.publish(u'crossbar.node.process.on_connection_start', state) returnValue(state) @inlineCallbacks def stop_connection(self, id, details=None): """ Stop a connection currently running within this process. :param id: The ID of the connection to stop. :type id: unicode :param details: Caller details. :type details: instance of :class:`autobahn.wamp.types.CallDetails` :returns dict -- A dict with component start information. """ self.log.debug("stop_connection: id={id}", id=id) if id not in self._connections: raise ApplicationError(u'crossbar.error.no_such_object', 'no connection with ID {} running in this process'.format(id)) connection = self._connections[id] try: yield connection.stop() except Exception as e: self.log.warn('could not stop connection {id}: {error}', error=e) raise del self._connections[id] state = connection.marshal() self.publish(u'crossbar.node.process.on_connection_stop', state) returnValue(state) def get_connections(self, details=None): """ Get connections currently running within this processs. :param details: Caller details. :type details: instance of :class:`autobahn.wamp.types.CallDetails` :returns list -- List of connections. """ self.log.debug("get_connections") res = [] for c in self._connections.values(): res.append(c.marshal()) return res def get_process_info(self, details=None): """ Get process information (open files, sockets, ...). :returns: dict -- Dictionary with process information. """ self.log.debug("{cls}.get_process_info", cls=self.__class__.__name__) if self._pinfo: return self._pinfo.get_info() else: emsg = "Could not retrieve process statistics: required packages not installed" raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) def get_process_stats(self, details=None): """ Get process statistics (CPU, memory, I/O). :returns: dict -- Dictionary with process statistics. """ self.log.debug("{cls}.get_process_stats", cls=self.__class__.__name__) if self._pinfo: return self._pinfo.get_stats() else: emsg = "Could not retrieve process statistics: required packages not installed" raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) def set_process_stats_monitoring(self, interval, details=None): """ Enable/disable periodic publication of process statistics. :param interval: The monitoring interval in seconds. Set to 0 to disable monitoring. :type interval: float """ self.log.debug("{cls}.set_process_stats_monitoring(interval = {interval})", cls=self.__class__.__name__, interval=interval) if self._pinfo: stats_monitor_set_topic = '{}.on_process_stats_monitoring_set'.format(self._uri_prefix) # stop and remove any existing monitor if self._pinfo_monitor: self._pinfo_monitor.stop() self._pinfo_monitor = None self.publish(stats_monitor_set_topic, 0, options=PublishOptions(exclude=details.caller)) # possibly start a new monitor if interval > 0: stats_topic = '{}.on_process_stats'.format(self._uri_prefix) def publish_stats(): stats = self._pinfo.get_stats() self._pinfo_monitor_seq += 1 stats[u'seq'] = self._pinfo_monitor_seq self.publish(stats_topic, stats) self._pinfo_monitor = LoopingCall(publish_stats) self._pinfo_monitor.start(interval) self.publish(stats_monitor_set_topic, interval, options=PublishOptions(exclude=details.caller)) else: emsg = "Cannot setup process statistics monitor: required packages not installed" raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) def trigger_gc(self, details=None): """ Manually trigger a garbage collection in this native process. This procedure is registered under ``crossbar.node.<node_id>.worker.<worker_id>.trigger_gc`` for native workers and under ``crossbar.node.<node_id>.controller.trigger_gc`` for node controllers. The procedure will publish an event when the garabage collection has finished to ``crossbar.node.<node_id>.worker.<worker_id>.on_gc_finished`` for native workers and ``crossbar.node.<node_id>.controller.on_gc_finished`` for node controllers: .. code-block:: javascript { "requester": { "session_id": 982734923, "auth_id": "bob", "auth_role": "admin" }, "duration": 190 } .. note:: The caller of this procedure will NOT receive the event. :returns: Time (wall clock) consumed for garbage collection in ms. :rtype: int """ self.log.debug("{cls}.trigger_gc", cls=self.__class__.__name__) started = rtime() # now trigger GC .. this is blocking! gc.collect() duration = int(round(1000. * (rtime() - started))) on_gc_finished = u'{}.on_gc_finished'.format(self._uri_prefix) self.publish( on_gc_finished, { u'requester': { u'session_id': details.caller, # FIXME: u'auth_id': None, u'auth_role': None }, u'duration': duration }, options=PublishOptions(exclude=details.caller) ) return duration @inlineCallbacks def start_manhole(self, config, details=None): """ Start a Manhole service within this process. **Usage:** This procedure is registered under * ``crossbar.node.<node_id>.worker.<worker_id>.start_manhole`` - for native workers * ``crossbar.node.<node_id>.controller.start_manhole`` - for node controllers The procedure takes a Manhole service configuration which defines a listening endpoint for the service and a list of users including passwords, e.g. .. code-block:: javascript { "endpoint": { "type": "tcp", "port": 6022 }, "users": [ { "user": "******", "password": "******" } ] } **Errors:** The procedure may raise the following errors: * ``crossbar.error.invalid_configuration`` - the provided configuration is invalid * ``crossbar.error.already_started`` - the Manhole service is already running (or starting) * ``crossbar.error.feature_unavailable`` - the required support packages are not installed **Events:** The procedure will publish an event when the service **is starting** to * ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_starting`` - for native workers * ``crossbar.node.<node_id>.controller.on_manhole_starting`` - for node controllers and publish an event when the service **has started** to * ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_started`` - for native workers * ``crossbar.node.<node_id>.controller.on_manhole_started`` - for node controllers :param config: Manhole service configuration. :type config: dict """ self.log.debug("{cls}.start_manhole(config = {config})", cls=self.__class__.__name__, config=config) if not _HAS_MANHOLE: emsg = "Could not start manhole: required packages are missing ({})".format(_MANHOLE_MISSING_REASON) self.log.error(emsg) raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) if self._manhole_service: emsg = "Could not start manhole - already running (or starting)" self.log.warn(emsg) raise ApplicationError(u"crossbar.error.already_started", emsg) try: checkconfig.check_manhole(config) except Exception as e: emsg = "Could not start manhole: invalid configuration ({})".format(e) self.log.error(emsg) raise ApplicationError(u'crossbar.error.invalid_configuration', emsg) # setup user authentication # checker = checkers.InMemoryUsernamePasswordDatabaseDontUse() for user in config['users']: checker.addUser(user['user'], user['password']) # setup manhole namespace # namespace = {'session': self} class PatchedTerminalSession(TerminalSession): # get rid of # exceptions.AttributeError: TerminalSession instance has no attribute 'windowChanged' def windowChanged(self, winSize): pass rlm = TerminalRealm() rlm.sessionFactory = PatchedTerminalSession # monkey patch rlm.chainedProtocolFactory.protocolFactory = lambda _: ColoredManhole(namespace) ptl = portal.Portal(rlm, [checker]) factory = ConchFactory(ptl) factory.noisy = False self._manhole_service = ManholeService(config, details.caller) starting_topic = '{}.on_manhole_starting'.format(self._uri_prefix) starting_info = self._manhole_service.marshal() # the caller gets a progressive result .. if details.progress: details.progress(starting_info) # .. while all others get an event self.publish(starting_topic, starting_info, options=PublishOptions(exclude=details.caller)) try: self._manhole_service.port = yield create_listening_port_from_config(config['endpoint'], self.cbdir, factory, self._reactor, self.log) except Exception as e: self._manhole_service = None emsg = "Manhole service endpoint cannot listen: {}".format(e) self.log.error(emsg) raise ApplicationError(u"crossbar.error.cannot_listen", emsg) # alright, manhole has started self._manhole_service.started = datetime.utcnow() self._manhole_service.status = 'started' started_topic = '{}.on_manhole_started'.format(self._uri_prefix) started_info = self._manhole_service.marshal() self.publish(started_topic, started_info, options=PublishOptions(exclude=details.caller)) returnValue(started_info) @inlineCallbacks def stop_manhole(self, details=None): """ Stop the Manhole service running in this process. This procedure is registered under * ``crossbar.node.<node_id>.worker.<worker_id>.stop_manhole`` for native workers and under * ``crossbar.node.<node_id>.controller.stop_manhole`` for node controllers When no Manhole service is currently running within this process, or the Manhole service is already shutting down, a ``crossbar.error.not_started`` WAMP error is raised. The procedure will publish an event when the service **is stopping** to * ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_stopping`` for native workers and * ``crossbar.node.<node_id>.controller.on_manhole_stopping`` for node controllers and will publish an event when the service **has stopped** to * ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_stopped`` for native workers and * ``crossbar.node.<node_id>.controller.on_manhole_stopped`` for node controllers """ self.log.debug("{cls}.stop_manhole", cls=self.__class__.__name__) if not self._manhole_service or self._manhole_service.status != 'started': emsg = "Cannot stop manhole: not running (or already shutting down)" raise ApplicationError(u"crossbar.error.not_started", emsg) self._manhole_service.status = 'stopping' stopping_topic = u'{}.on_manhole_stopping'.format(self._uri_prefix) stopping_info = None # the caller gets a progressive result .. if details.progress: details.progress(stopping_info) # .. while all others get an event self.publish(stopping_topic, stopping_info, options=PublishOptions(exclude=details.caller)) try: yield self._manhole_service.port.stopListening() except Exception as e: self.log.warn("error while stop listening on endpoint: {error}", error=e) self._manhole_service = None stopped_topic = u'{}.on_manhole_stopped'.format(self._uri_prefix) stopped_info = None self.publish(stopped_topic, stopped_info, options=PublishOptions(exclude=details.caller)) returnValue(stopped_info) def get_manhole(self, details=None): """ Get current manhole service information. :returns: dict -- A dict with service information or `None` if the service is not running. """ self.log.debug("{cls}.get_manhole", cls=self.__class__.__name__) if not _HAS_MANHOLE: emsg = "Could not start manhole: required packages are missing ({})".format(_MANHOLE_MISSING_REASON) self.log.error(emsg) raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) if not self._manhole_service: return None else: return self._manhole_service.marshal() def utcnow(self, details=None): """ Return current time as determined from within this process. **Usage:** This procedure is registered under * ``crossbar.node.<node_id>.worker.<worker_id>.utcnow`` for native workers and under * ``crossbar.node.<node_id>.controller.utcnow`` for node controllers :returns: Current time (UTC) in UTC ISO 8601 format. :rtype: unicode """ self.log.debug("{cls}.utcnow", cls=self.__class__.__name__) return utcnow() def started(self, details=None): """ Return start time of this process. **Usage:** This procedure is registered under * ``crossbar.node.<node_id>.worker.<worker_id>.started`` for native workers and under * ``crossbar.node.<node_id>.controller.started`` for node controllers :returns: Start time (UTC) in UTC ISO 8601 format. :rtype: unicode """ self.log.debug("{cls}.started", cls=self.__class__.__name__) return utcstr(self._started) def uptime(self, details=None): """ Return uptime of this process. **Usage:** This procedure is registered under * ``crossbar.node.<node_id>.worker.<worker_id>.uptime`` for native workers and under * ``crossbar.node.<node_id>.controller.uptime`` for node controllers :returns: Uptime in seconds. :rtype: float """ self.log.debug("{cls}.uptime", cls=self.__class__.__name__) now = datetime.utcnow() return (now - self._started).total_seconds()
class AppSession(ApplicationSession): @inlineCallbacks def onJoin(self, details): self._db = Database(dbpath='../results') self._schema = Schema.attach(self._db) self._pinfo = ProcessInfo() self._logname = self.config.extra['logname'] self._period = self.config.extra.get('period', 5.) self._running = True dl = [] for i in range(8): d = self._loop(i) dl.append(d) d = gatherResults(dl) try: yield d except TransportLost: pass def onLeave(self, details): self._running = False @inlineCallbacks def _loop(self, index): prefix = self.config.extra['prefix'] last = None while self._running: rtts = [] batch_started_str = utcnow() batch_started = time() while (time() - batch_started) < self._period: ts_req = time_ns() res = yield self.call('{}.echo'.format(prefix), ts_req) ts_res = time_ns() assert res == ts_req rtt = ts_res - ts_req rtts.append(rtt) stats = self._pinfo.get_stats() if last: batch_duration = (stats['time'] - last['time']) / 10**9 ctx = round( (stats['voluntary'] - last['voluntary']) / batch_duration, 0) self.log.info('{logname}: {cpu} cpu, {mem} mem, {ctx} ctx', logname=self._logname, cpu=round(stats['cpu_percent'], 1), mem=round(stats['mem_percent'], 1), ctx=ctx) rtts = sorted(rtts) sr = WampStatsRecord() sr.key = '{}#{}.{}'.format(batch_started_str, self._logname, index) sr.count = len(rtts) sr.calls_per_sec = int(round(sr.count / batch_duration, 0)) # all times here are in microseconds: sr.avg_rtt = round(1000000. * batch_duration / float(sr.count), 1) sr.max_rtt = round(rtts[-1] / 1000, 1) sr.q50_rtt = round(rtts[int(sr.count / 2.)] / 1000, 1) sr.q99_rtt = round(rtts[int(-(sr.count / 100.))] / 1000, 1) sr.q995_rtt = round(rtts[int(-(sr.count / 995.))] / 1000, 1) with self._db.begin(write=True) as txn: self._schema.wamp_stats[txn, sr.key] = sr print( "{}: {} calls, {} calls/sec, RTT (us): q50 {}, avg {}, q99 {}, q995 {}, max {}" .format(sr.key, sr.count, sr.calls_per_sec, sr.q50_rtt, sr.avg_rtt, sr.q99_rtt, sr.q995_rtt, sr.max_rtt)) last = stats
class NativeProcess(ApplicationSession): """ A native Crossbar.io process (currently: controller, router or container). """ log = make_logger() WORKER_TYPE = u'native' def onUserError(self, fail, msg): """ Implements :func:`autobahn.wamp.interfaces.ISession.onUserError` """ if isinstance(fail.value, ApplicationError): self.log.debug('{klass}.onUserError(): "{msg}"', klass=self.__class__.__name__, msg=fail.value.error_message()) else: self.log.error( '{klass}.onUserError(): "{msg}"\n{traceback}', klass=self.__class__.__name__, msg=msg, traceback=txaio.failure_format_traceback(fail), ) def __init__(self, config=None, reactor=None, personality=None): # Twisted reactor if not reactor: from twisted.internet import reactor self._reactor = reactor self._reactor = reactor # node/software personality if personality: self.personality = personality else: from crossbar.personality import Personality self.personality = Personality self._node_id = config.extra.node if config and config.extra else None self._worker_id = config.extra.worker if config and config.extra else None self._uri_prefix = u'crossbar.worker.{}'.format(self._worker_id) # base ctor super(ApplicationSession, self).__init__(config=config) def onConnect(self, do_join=True): if not hasattr(self, 'cbdir'): self.cbdir = self.config.extra.cbdir if not hasattr(self, '_uri_prefix'): self._uri_prefix = 'crossbar.node.{}'.format(self.config.extra.node) self._started = datetime.utcnow() # see: BaseSession self.include_traceback = False self._manhole_service = None if _HAS_PSUTIL: self._pinfo = ProcessInfo() self._pmonitor = ProcessMonitor(self.WORKER_TYPE, {}) self._pinfo_monitor = None self._pinfo_monitor_seq = 0 else: self._pinfo = None self._pmonitor = None self._pinfo_monitor = None self._pinfo_monitor_seq = None self.log.info("Process utilities not available") if do_join: self.join(self.config.realm) @inlineCallbacks def onJoin(self, details): """ Called when process has joined the node's management realm. """ regs = yield self.register( self, prefix=u'{}.'.format(self._uri_prefix), options=RegisterOptions(details_arg='details'), ) self.log.info("Registered {len_reg} procedures", len_reg=len(regs)) for reg in regs: if isinstance(reg, Failure): self.log.error("Failed to register: {f}", f=reg, log_failure=reg) else: self.log.debug(' {proc}', proc=reg.procedure) returnValue(regs) @wamp.register(None) def get_cpu_count(self, logical=True, details=None): """ Returns the CPU core count on the machine this process is running on. :param logical: If enabled (default), include logical CPU cores ("Hyperthreading"), else only count physical CPU cores. :type logical: bool :returns: The number of CPU cores. :rtype: int """ if not _HAS_PSUTIL: emsg = "unable to get CPU count: required package 'psutil' is not installed" self.log.warn(emsg) raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) return psutil.cpu_count(logical=logical) @wamp.register(None) def get_cpus(self, details=None): """ :returns: List of CPU IDs. :rtype: list[int] """ if not _HAS_PSUTIL: emsg = "unable to get CPUs: required package 'psutil' is not installed" self.log.warn(emsg) raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) return self._pinfo.cpus @wamp.register(None) def get_cpu_affinity(self, details=None): """ Get CPU affinity of this process. :returns: List of CPU IDs the process affinity is set to. :rtype: list[int] """ if not _HAS_PSUTIL: emsg = "unable to get CPU affinity: required package 'psutil' is not installed" self.log.warn(emsg) raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) try: p = psutil.Process(os.getpid()) current_affinity = p.cpu_affinity() except Exception as e: emsg = "Could not get CPU affinity: {}".format(e) self.log.failure(emsg) raise ApplicationError(u"crossbar.error.runtime_error", emsg) else: return current_affinity @wamp.register(None) def set_cpu_affinity(self, cpus, relative=True, details=None): """ Set CPU affinity of this process. :param cpus: List of CPU IDs to set process affinity to. Each CPU ID must be from the list `[0 .. N_CPUs]`, where N_CPUs can be retrieved via ``crossbar.worker.<worker_id>.get_cpu_count``. :type cpus: list[int] :returns: List of CPU IDs the process affinity is set to. :rtype: list[int] """ if not _HAS_PSUTIL: emsg = "Unable to set CPU affinity: required package 'psutil' is not installed" self.log.warn(emsg) raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) if sys.platform.startswith('darwin'): # https://superuser.com/questions/149312/how-to-set-processor-affinity-on-os-x emsg = "Unable to set CPU affinity: OSX lacks process CPU affinity" self.log.warn(emsg) raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) if relative: _cpu_ids = self._pinfo.cpus _cpus = [_cpu_ids[i] for i in cpus] else: _cpus = cpus try: p = psutil.Process(os.getpid()) p.cpu_affinity(_cpus) new_affinity = p.cpu_affinity() if set(_cpus) != set(new_affinity): raise Exception('CPUs mismatch after affinity setting ({} != {})'.format(set(_cpus), set(new_affinity))) except Exception as e: emsg = "Could not set CPU affinity: {}".format(e) self.log.failure(emsg) raise ApplicationError(u"crossbar.error.runtime_error", emsg) else: # publish info to all but the caller .. # cpu_affinity_set_topic = u'{}.on_cpu_affinity_set'.format(self._uri_prefix) cpu_affinity_set_info = { u'cpus': cpus, u'relative': relative, u'affinity': new_affinity, u'who': details.caller } self.publish(cpu_affinity_set_topic, cpu_affinity_set_info, options=PublishOptions(exclude=details.caller)) # .. and return info directly to caller # return new_affinity @wamp.register(None) def get_process_info(self, details=None): """ Get process information (open files, sockets, ...). :returns: Dictionary with process information. """ self.log.debug("{cls}.get_process_info", cls=self.__class__.__name__) if self._pinfo: # psutil.AccessDenied # PermissionError: [Errno 13] Permission denied: '/proc/14787/io' return self._pinfo.get_info() else: emsg = "Could not retrieve process statistics: required packages not installed" raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) @wamp.register(None) def get_process_stats(self, details=None): """ Get process statistics (CPU, memory, I/O). :returns: Dictionary with process statistics. """ self.log.debug("{cls}.get_process_stats", cls=self.__class__.__name__) if self._pinfo: return self._pinfo.get_stats() else: emsg = "Could not retrieve process statistics: required packages not installed" raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) @wamp.register(None) def get_process_monitor(self, details=None): self.log.debug("{cls}.get_process_monitor", cls=self.__class__.__name__) if self._pmonitor: return self._pmonitor.poll() else: emsg = "Could not retrieve process statistics: required packages not installed" raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) @wamp.register(None) def set_process_stats_monitoring(self, interval, details=None): """ Enable/disable periodic publication of process statistics. :param interval: The monitoring interval in seconds. Set to 0 to disable monitoring. :type interval: float """ self.log.debug("{cls}.set_process_stats_monitoring(interval = {interval})", cls=self.__class__.__name__, interval=interval) if self._pinfo: stats_monitor_set_topic = '{}.on_process_stats_monitoring_set'.format(self._uri_prefix) # stop and remove any existing monitor if self._pinfo_monitor: self._pinfo_monitor.stop() self._pinfo_monitor = None self.publish(stats_monitor_set_topic, 0, options=PublishOptions(exclude=details.caller)) # possibly start a new monitor if interval > 0: stats_topic = '{}.on_process_stats'.format(self._uri_prefix) def publish_stats(): stats = self._pinfo.get_stats() self._pinfo_monitor_seq += 1 stats[u'seq'] = self._pinfo_monitor_seq self.publish(stats_topic, stats) self._pinfo_monitor = LoopingCall(publish_stats) self._pinfo_monitor.start(interval) self.publish(stats_monitor_set_topic, interval, options=PublishOptions(exclude=details.caller)) else: emsg = "Cannot setup process statistics monitor: required packages not installed" raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) @wamp.register(None) def trigger_gc(self, details=None): """ Manually trigger a garbage collection in this native process. This procedure is registered under ``crossbar.node.<node_id>.worker.<worker_id>.trigger_gc`` for native workers and under ``crossbar.node.<node_id>.controller.trigger_gc`` for node controllers. The procedure will publish an event when the garabage collection has finished to ``crossbar.node.<node_id>.worker.<worker_id>.on_gc_finished`` for native workers and ``crossbar.node.<node_id>.controller.on_gc_finished`` for node controllers: .. code-block:: javascript { "requester": { "session_id": 982734923, "auth_id": "bob", "auth_role": "admin" }, "duration": 190 } .. note:: The caller of this procedure will NOT receive the event. :returns: Time (wall clock) consumed for garbage collection in ms. :rtype: int """ self.log.debug("{cls}.trigger_gc", cls=self.__class__.__name__) started = rtime() # now trigger GC .. this is blocking! gc.collect() duration = int(round(1000. * (rtime() - started))) on_gc_finished = u'{}.on_gc_finished'.format(self._uri_prefix) self.publish( on_gc_finished, { u'requester': { u'session_id': details.caller, # FIXME: u'auth_id': None, u'auth_role': None }, u'duration': duration }, options=PublishOptions(exclude=details.caller) ) return duration @wamp.register(None) @inlineCallbacks def start_manhole(self, config, details=None): """ Start a Manhole service within this process. **Usage:** This procedure is registered under * ``crossbar.node.<node_id>.worker.<worker_id>.start_manhole`` - for native workers * ``crossbar.node.<node_id>.controller.start_manhole`` - for node controllers The procedure takes a Manhole service configuration which defines a listening endpoint for the service and a list of users including passwords, e.g. .. code-block:: javascript { "endpoint": { "type": "tcp", "port": 6022 }, "users": [ { "user": "******", "password": "******" } ] } **Errors:** The procedure may raise the following errors: * ``crossbar.error.invalid_configuration`` - the provided configuration is invalid * ``crossbar.error.already_started`` - the Manhole service is already running (or starting) * ``crossbar.error.feature_unavailable`` - the required support packages are not installed **Events:** The procedure will publish an event when the service **is starting** to * ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_starting`` - for native workers * ``crossbar.node.<node_id>.controller.on_manhole_starting`` - for node controllers and publish an event when the service **has started** to * ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_started`` - for native workers * ``crossbar.node.<node_id>.controller.on_manhole_started`` - for node controllers :param config: Manhole service configuration. :type config: dict """ self.log.debug("{cls}.start_manhole(config = {config})", cls=self.__class__.__name__, config=config) if not _HAS_MANHOLE: emsg = "Could not start manhole: required packages are missing ({})".format(_MANHOLE_MISSING_REASON) self.log.error(emsg) raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) if self._manhole_service: emsg = "Could not start manhole - already running (or starting)" self.log.warn(emsg) raise ApplicationError(u"crossbar.error.already_started", emsg) try: self.personality.check_manhole(self.personality, config) except Exception as e: emsg = "Could not start manhole: invalid configuration ({})".format(e) self.log.error(emsg) raise ApplicationError(u'crossbar.error.invalid_configuration', emsg) from twisted.conch.ssh import keys from twisted.conch.manhole_ssh import ( ConchFactory, TerminalRealm, TerminalSession) from twisted.conch.manhole import ColoredManhole from twisted.conch.checkers import SSHPublicKeyDatabase class PublicKeyChecker(SSHPublicKeyDatabase): def __init__(self, userKeys): self.userKeys = {} for username, keyData in userKeys.items(): self.userKeys[username] = keys.Key.fromString(data=keyData).blob() def checkKey(self, credentials): username = credentials.username.decode('utf8') if username in self.userKeys: keyBlob = self.userKeys[username] return keyBlob == credentials.blob # setup user authentication # authorized_keys = { 'oberstet': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCz7K1QwDhaq/Bi8o0uqiJQuVFCDQL5rbRvMClLHRx9KE3xP2Fh2eapzXuYGSgtG9Fyz1UQd+1oNM3wuNnT/DsBUBQrECP4bpFIHcJkMaFTARlCagkXosWsadzNnkW0osUCuHYMrzBJuXWF2GH+0OFCtVu+8E+4Mhvchu9xsHG8PM92SpI6aP0TtmT9D/0Bsm9JniRj8kndeS+iWG4s/pEGj7Rg7eGnbyQJt/9Jc1nWl6PngGbwp63dMVmh+8LP49PtfnxY8m9fdwpL4oW9U8beYqm8hyfBPN2yDXaehg6RILjIa7LU2/6bu96ZgnIz26zi/X9XlnJQt2aahWJs1+GR oberstet@thinkpad-t430s' } checker = PublicKeyChecker(authorized_keys) # setup manhole namespace # namespace = {'session': self} class PatchedTerminalSession(TerminalSession): # get rid of # exceptions.AttributeError: TerminalSession instance has no attribute 'windowChanged' def windowChanged(self, winSize): pass rlm = TerminalRealm() rlm.sessionFactory = PatchedTerminalSession # monkey patch rlm.chainedProtocolFactory.protocolFactory = lambda _: ColoredManhole(namespace) ptl = portal.Portal(rlm) ptl.registerChecker(checker) factory = ConchFactory(ptl) factory.noisy = False private_key = keys.Key.fromFile(os.path.join(self.cbdir, 'ssh_host_rsa_key')) public_key = private_key.public() publicKeys = { b'ssh-rsa': public_key } privateKeys = { b'ssh-rsa': private_key } factory.publicKeys = publicKeys factory.privateKeys = privateKeys self._manhole_service = ManholeService(config, details.caller) starting_topic = '{}.on_manhole_starting'.format(self._uri_prefix) starting_info = self._manhole_service.marshal() # the caller gets a progressive result .. if details.progress: details.progress(starting_info) # .. while all others get an event self.publish(starting_topic, starting_info, options=PublishOptions(exclude=details.caller)) try: self._manhole_service.port = yield create_listening_port_from_config(config['endpoint'], self.cbdir, factory, self._reactor, self.log) except Exception as e: self._manhole_service = None emsg = "Manhole service endpoint cannot listen: {}".format(e) self.log.error(emsg) raise ApplicationError(u"crossbar.error.cannot_listen", emsg) # alright, manhole has started self._manhole_service.started = datetime.utcnow() self._manhole_service.status = 'started' started_topic = '{}.on_manhole_started'.format(self._uri_prefix) started_info = self._manhole_service.marshal() self.publish(started_topic, started_info, options=PublishOptions(exclude=details.caller)) returnValue(started_info) @wamp.register(None) @inlineCallbacks def stop_manhole(self, details=None): """ Stop the Manhole service running in this process. This procedure is registered under * ``crossbar.node.<node_id>.worker.<worker_id>.stop_manhole`` for native workers and under * ``crossbar.node.<node_id>.controller.stop_manhole`` for node controllers When no Manhole service is currently running within this process, or the Manhole service is already shutting down, a ``crossbar.error.not_started`` WAMP error is raised. The procedure will publish an event when the service **is stopping** to * ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_stopping`` for native workers and * ``crossbar.node.<node_id>.controller.on_manhole_stopping`` for node controllers and will publish an event when the service **has stopped** to * ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_stopped`` for native workers and * ``crossbar.node.<node_id>.controller.on_manhole_stopped`` for node controllers """ self.log.debug("{cls}.stop_manhole", cls=self.__class__.__name__) if not self._manhole_service or self._manhole_service.status != 'started': emsg = "Cannot stop manhole: not running (or already shutting down)" raise ApplicationError(u"crossbar.error.not_started", emsg) self._manhole_service.status = 'stopping' stopping_topic = u'{}.on_manhole_stopping'.format(self._uri_prefix) stopping_info = None # the caller gets a progressive result .. if details.progress: details.progress(stopping_info) # .. while all others get an event self.publish(stopping_topic, stopping_info, options=PublishOptions(exclude=details.caller)) try: yield self._manhole_service.port.stopListening() except Exception as e: self.log.warn("error while stop listening on endpoint: {error}", error=e) self._manhole_service = None stopped_topic = u'{}.on_manhole_stopped'.format(self._uri_prefix) stopped_info = None self.publish(stopped_topic, stopped_info, options=PublishOptions(exclude=details.caller)) returnValue(stopped_info) @wamp.register(None) def get_manhole(self, details=None): """ Get current manhole service information. :returns: A dict with service information or `None` if the service is not running. """ self.log.debug("{cls}.get_manhole", cls=self.__class__.__name__) if not _HAS_MANHOLE: emsg = "Could not start manhole: required packages are missing ({})".format(_MANHOLE_MISSING_REASON) self.log.error(emsg) raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) if not self._manhole_service: return None else: return self._manhole_service.marshal() @wamp.register(None) def utcnow(self, details=None): """ Return current time as determined from within this process. **Usage:** This procedure is registered under * ``crossbar.node.<node_id>.worker.<worker_id>.utcnow`` for native workers and under * ``crossbar.node.<node_id>.controller.utcnow`` for node controllers :returns: Current time (UTC) in UTC ISO 8601 format. :rtype: str """ self.log.debug("{cls}.utcnow", cls=self.__class__.__name__) return utcnow() @wamp.register(None) def started(self, details=None): """ Return start time of this process. **Usage:** This procedure is registered under * ``crossbar.node.<node_id>.worker.<worker_id>.started`` for native workers and under * ``crossbar.node.<node_id>.controller.started`` for node controllers :returns: Start time (UTC) in UTC ISO 8601 format. :rtype: str """ self.log.debug("{cls}.started", cls=self.__class__.__name__) return utcstr(self._started) @wamp.register(None) def uptime(self, details=None): """ Return uptime of this process. **Usage:** This procedure is registered under * ``crossbar.node.<node_id>.worker.<worker_id>.uptime`` for native workers and under * ``crossbar.node.<node_id>.controller.uptime`` for node controllers :returns: Uptime in seconds. :rtype: float """ self.log.debug("{cls}.uptime", cls=self.__class__.__name__) now = datetime.utcnow() return (now - self._started).total_seconds()
class NativeProcess(ApplicationSession): """ A native Crossbar.io process (currently: controller, router or container). """ log = make_logger() def __init__(self, config=None, reactor=None, personality=None): # Twisted reactor if not reactor: from twisted.internet import reactor self._reactor = reactor self._reactor = reactor # node/software personality if personality: self.personality = personality else: from crossbar.personality import Personality self.personality = Personality # base ctor super(ApplicationSession, self).__init__(config=config) def onConnect(self, do_join=True): """ """ if not hasattr(self, 'cbdir'): self.cbdir = self.config.extra.cbdir if not hasattr(self, '_uri_prefix'): self._uri_prefix = 'crossbar.node.{}'.format( self.config.extra.node) self._started = datetime.utcnow() # see: BaseSession self.include_traceback = False self._manhole_service = None if _HAS_PSUTIL: self._pinfo = ProcessInfo() self._pinfo_monitor = None self._pinfo_monitor_seq = 0 else: self._pinfo = None self._pinfo_monitor = None self._pinfo_monitor_seq = None self.log.info("Process utilities not available") self._connections = {} if do_join: self.join(self.config.realm) @inlineCallbacks def onJoin(self, details): """ Called when process has joined the node's management realm. """ regs = yield self.register( self, prefix=u'{}.'.format(self._uri_prefix), options=RegisterOptions(details_arg='details'), ) self.log.info("Registered {len_reg} procedures", len_reg=len(regs)) for reg in regs: if isinstance(reg, Failure): self.log.error("Failed to register: {f}", f=reg, log_failure=reg) else: self.log.debug(' {proc}', proc=reg.procedure) returnValue(regs) @wamp.register(None) def get_cpu_count(self, logical=True, details=None): """ Returns the CPU core count on the machine this process is running on. :param logical: If enabled (default), include logical CPU cores ("Hyperthreading"), else only count physical CPU cores. :type logical: bool :returns: The number of CPU cores. :rtype: int """ if not _HAS_PSUTIL: emsg = "unable to get CPU count: required package 'psutil' is not installed" self.log.warn(emsg) raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) return psutil.cpu_count(logical=logical) @wamp.register(None) def get_cpus(self, details=None): """ :returns: List of CPU IDs. :rtype: list[Int] """ if not _HAS_PSUTIL: emsg = "unable to get CPUs: required package 'psutil' is not installed" self.log.warn(emsg) raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) return self._pinfo.cpus @wamp.register(None) def get_cpu_affinity(self, details=None): """ Get CPU affinity of this process. :returns: List of CPU IDs the process affinity is set to. :rtype: list of int """ if not _HAS_PSUTIL: emsg = "unable to get CPU affinity: required package 'psutil' is not installed" self.log.warn(emsg) raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) try: p = psutil.Process(os.getpid()) current_affinity = p.cpu_affinity() except Exception as e: emsg = "Could not get CPU affinity: {}".format(e) self.log.failure(emsg) raise ApplicationError(u"crossbar.error.runtime_error", emsg) else: return current_affinity @wamp.register(None) def set_cpu_affinity(self, cpus, relative=True, details=None): """ Set CPU affinity of this process. :param cpus: List of CPU IDs to set process affinity to. Each CPU ID must be from the list `[0 .. N_CPUs]`, where N_CPUs can be retrieved via ``crossbar.worker.<worker_id>.get_cpu_count``. :type cpus: list of int :returns: List of CPU IDs the process affinity is set to. :rtype: list of int """ if not _HAS_PSUTIL: emsg = "Unable to set CPU affinity: required package 'psutil' is not installed" self.log.warn(emsg) raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) if relative: _cpu_ids = self._pinfo.cpus _cpus = [_cpu_ids[i] for i in cpus] else: _cpus = cpus try: p = psutil.Process(os.getpid()) p.cpu_affinity(_cpus) new_affinity = p.cpu_affinity() if set(_cpus) != set(new_affinity): raise Exception( 'CPUs mismatch after affinity setting ({} != {})'.format( set(_cpus), set(new_affinity))) except Exception as e: emsg = "Could not set CPU affinity: {}".format(e) self.log.failure(emsg) raise ApplicationError(u"crossbar.error.runtime_error", emsg) else: # publish info to all but the caller .. # cpu_affinity_set_topic = u'{}.on_cpu_affinity_set'.format( self._uri_prefix) cpu_affinity_set_info = { u'cpus': cpus, u'relative': relative, u'affinity': new_affinity, u'who': details.caller } self.publish(cpu_affinity_set_topic, cpu_affinity_set_info, options=PublishOptions(exclude=details.caller)) # .. and return info directly to caller # return new_affinity @inlineCallbacks def start_connection(self, id, config, details=None): """ Starts a connection in this process. :param id: The ID for the started connection. :type id: unicode :param config: Connection configuration. :type config: dict :param details: Caller details. :type details: instance of :class:`autobahn.wamp.types.CallDetails` :returns dict -- The connection. """ self.log.debug("start_connection: id={id}, config={config}", id=id, config=config) # prohibit starting a component twice # if id in self._connections: emsg = "cannot start connection: a connection with id={} is already started".format( id) self.log.warn(emsg) raise ApplicationError(u"crossbar.error.invalid_configuration", emsg) # check configuration # try: self.personality.check_connection(self.personality, config) except Exception as e: emsg = "invalid connection configuration ({})".format(e) self.log.warn(emsg) raise ApplicationError(u"crossbar.error.invalid_configuration", emsg) else: self.log.info("Starting {ptype} in process.", ptype=config['type']) if config['type'] == u'postgresql.connection': if _HAS_POSTGRESQL: connection = PostgreSQLConnection(id, config) else: emsg = "unable to start connection - required PostgreSQL driver package not installed" self.log.warn(emsg) raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) else: # should not arrive here raise Exception("logic error") self._connections[id] = connection try: yield connection.start() self.log.info( "Connection {connection_type} started '{connection_id}'", connection_id=id, connection_type=config['type']) except Exception as e: del self._connections[id] raise state = connection.marshal() self.publish(u'crossbar.node.process.on_connection_start', state) returnValue(state) @inlineCallbacks def stop_connection(self, id, details=None): """ Stop a connection currently running within this process. :param id: The ID of the connection to stop. :type id: unicode :param details: Caller details. :type details: instance of :class:`autobahn.wamp.types.CallDetails` :returns dict -- A dict with component start information. """ self.log.debug("stop_connection: id={id}", id=id) if id not in self._connections: raise ApplicationError( u'crossbar.error.no_such_object', 'no connection with ID {} running in this process'.format(id)) connection = self._connections[id] try: yield connection.stop() except Exception as e: self.log.warn('could not stop connection {id}: {error}', error=e) raise del self._connections[id] state = connection.marshal() self.publish(u'crossbar.node.process.on_connection_stop', state) returnValue(state) def get_connections(self, details=None): """ Get connections currently running within this processs. :param details: Caller details. :type details: instance of :class:`autobahn.wamp.types.CallDetails` :returns list -- List of connections. """ self.log.debug("get_connections") res = [] for c in self._connections.values(): res.append(c.marshal()) return res @wamp.register(None) def get_process_info(self, details=None): """ Get process information (open files, sockets, ...). :returns: dict -- Dictionary with process information. """ self.log.debug("{cls}.get_process_info", cls=self.__class__.__name__) if self._pinfo: # psutil.AccessDenied # PermissionError: [Errno 13] Permission denied: '/proc/14787/io' return self._pinfo.get_info() else: emsg = "Could not retrieve process statistics: required packages not installed" raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) @wamp.register(None) def get_process_stats(self, details=None): """ Get process statistics (CPU, memory, I/O). :returns: dict -- Dictionary with process statistics. """ self.log.debug("{cls}.get_process_stats", cls=self.__class__.__name__) if self._pinfo: return self._pinfo.get_stats() else: emsg = "Could not retrieve process statistics: required packages not installed" raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) @wamp.register(None) def set_process_stats_monitoring(self, interval, details=None): """ Enable/disable periodic publication of process statistics. :param interval: The monitoring interval in seconds. Set to 0 to disable monitoring. :type interval: float """ self.log.debug( "{cls}.set_process_stats_monitoring(interval = {interval})", cls=self.__class__.__name__, interval=interval) if self._pinfo: stats_monitor_set_topic = '{}.on_process_stats_monitoring_set'.format( self._uri_prefix) # stop and remove any existing monitor if self._pinfo_monitor: self._pinfo_monitor.stop() self._pinfo_monitor = None self.publish(stats_monitor_set_topic, 0, options=PublishOptions(exclude=details.caller)) # possibly start a new monitor if interval > 0: stats_topic = '{}.on_process_stats'.format(self._uri_prefix) def publish_stats(): stats = self._pinfo.get_stats() self._pinfo_monitor_seq += 1 stats[u'seq'] = self._pinfo_monitor_seq self.publish(stats_topic, stats) self._pinfo_monitor = LoopingCall(publish_stats) self._pinfo_monitor.start(interval) self.publish(stats_monitor_set_topic, interval, options=PublishOptions(exclude=details.caller)) else: emsg = "Cannot setup process statistics monitor: required packages not installed" raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) @wamp.register(None) def trigger_gc(self, details=None): """ Manually trigger a garbage collection in this native process. This procedure is registered under ``crossbar.node.<node_id>.worker.<worker_id>.trigger_gc`` for native workers and under ``crossbar.node.<node_id>.controller.trigger_gc`` for node controllers. The procedure will publish an event when the garabage collection has finished to ``crossbar.node.<node_id>.worker.<worker_id>.on_gc_finished`` for native workers and ``crossbar.node.<node_id>.controller.on_gc_finished`` for node controllers: .. code-block:: javascript { "requester": { "session_id": 982734923, "auth_id": "bob", "auth_role": "admin" }, "duration": 190 } .. note:: The caller of this procedure will NOT receive the event. :returns: Time (wall clock) consumed for garbage collection in ms. :rtype: int """ self.log.debug("{cls}.trigger_gc", cls=self.__class__.__name__) started = rtime() # now trigger GC .. this is blocking! gc.collect() duration = int(round(1000. * (rtime() - started))) on_gc_finished = u'{}.on_gc_finished'.format(self._uri_prefix) self.publish( on_gc_finished, { u'requester': { u'session_id': details.caller, # FIXME: u'auth_id': None, u'auth_role': None }, u'duration': duration }, options=PublishOptions(exclude=details.caller)) return duration @wamp.register(None) @inlineCallbacks def start_manhole(self, config, details=None): """ Start a Manhole service within this process. **Usage:** This procedure is registered under * ``crossbar.node.<node_id>.worker.<worker_id>.start_manhole`` - for native workers * ``crossbar.node.<node_id>.controller.start_manhole`` - for node controllers The procedure takes a Manhole service configuration which defines a listening endpoint for the service and a list of users including passwords, e.g. .. code-block:: javascript { "endpoint": { "type": "tcp", "port": 6022 }, "users": [ { "user": "******", "password": "******" } ] } **Errors:** The procedure may raise the following errors: * ``crossbar.error.invalid_configuration`` - the provided configuration is invalid * ``crossbar.error.already_started`` - the Manhole service is already running (or starting) * ``crossbar.error.feature_unavailable`` - the required support packages are not installed **Events:** The procedure will publish an event when the service **is starting** to * ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_starting`` - for native workers * ``crossbar.node.<node_id>.controller.on_manhole_starting`` - for node controllers and publish an event when the service **has started** to * ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_started`` - for native workers * ``crossbar.node.<node_id>.controller.on_manhole_started`` - for node controllers :param config: Manhole service configuration. :type config: dict """ self.log.debug("{cls}.start_manhole(config = {config})", cls=self.__class__.__name__, config=config) if not _HAS_MANHOLE: emsg = "Could not start manhole: required packages are missing ({})".format( _MANHOLE_MISSING_REASON) self.log.error(emsg) raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) if self._manhole_service: emsg = "Could not start manhole - already running (or starting)" self.log.warn(emsg) raise ApplicationError(u"crossbar.error.already_started", emsg) try: self.personality.check_manhole(self.personality, config) except Exception as e: emsg = "Could not start manhole: invalid configuration ({})".format( e) self.log.error(emsg) raise ApplicationError(u'crossbar.error.invalid_configuration', emsg) from twisted.conch.ssh import keys from twisted.conch.manhole_ssh import (ConchFactory, TerminalRealm, TerminalSession) from twisted.conch.manhole import ColoredManhole from twisted.conch.checkers import SSHPublicKeyDatabase class PublicKeyChecker(SSHPublicKeyDatabase): def __init__(self, userKeys): self.userKeys = {} for username, keyData in userKeys.items(): self.userKeys[username] = keys.Key.fromString( data=keyData).blob() def checkKey(self, credentials): username = credentials.username.decode('utf8') if username in self.userKeys: keyBlob = self.userKeys[username] return keyBlob == credentials.blob # setup user authentication # authorized_keys = { 'oberstet': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCz7K1QwDhaq/Bi8o0uqiJQuVFCDQL5rbRvMClLHRx9KE3xP2Fh2eapzXuYGSgtG9Fyz1UQd+1oNM3wuNnT/DsBUBQrECP4bpFIHcJkMaFTARlCagkXosWsadzNnkW0osUCuHYMrzBJuXWF2GH+0OFCtVu+8E+4Mhvchu9xsHG8PM92SpI6aP0TtmT9D/0Bsm9JniRj8kndeS+iWG4s/pEGj7Rg7eGnbyQJt/9Jc1nWl6PngGbwp63dMVmh+8LP49PtfnxY8m9fdwpL4oW9U8beYqm8hyfBPN2yDXaehg6RILjIa7LU2/6bu96ZgnIz26zi/X9XlnJQt2aahWJs1+GR oberstet@thinkpad-t430s' } checker = PublicKeyChecker(authorized_keys) # setup manhole namespace # namespace = {'session': self} class PatchedTerminalSession(TerminalSession): # get rid of # exceptions.AttributeError: TerminalSession instance has no attribute 'windowChanged' def windowChanged(self, winSize): pass rlm = TerminalRealm() rlm.sessionFactory = PatchedTerminalSession # monkey patch rlm.chainedProtocolFactory.protocolFactory = lambda _: ColoredManhole( namespace) ptl = portal.Portal(rlm) ptl.registerChecker(checker) factory = ConchFactory(ptl) factory.noisy = False private_key = keys.Key.fromFile( os.path.join(self.cbdir, 'ssh_host_rsa_key')) public_key = private_key.public() publicKeys = {b'ssh-rsa': public_key} privateKeys = {b'ssh-rsa': private_key} factory.publicKeys = publicKeys factory.privateKeys = privateKeys self._manhole_service = ManholeService(config, details.caller) starting_topic = '{}.on_manhole_starting'.format(self._uri_prefix) starting_info = self._manhole_service.marshal() # the caller gets a progressive result .. if details.progress: details.progress(starting_info) # .. while all others get an event self.publish(starting_topic, starting_info, options=PublishOptions(exclude=details.caller)) try: self._manhole_service.port = yield create_listening_port_from_config( config['endpoint'], self.cbdir, factory, self._reactor, self.log) except Exception as e: self._manhole_service = None emsg = "Manhole service endpoint cannot listen: {}".format(e) self.log.error(emsg) raise ApplicationError(u"crossbar.error.cannot_listen", emsg) # alright, manhole has started self._manhole_service.started = datetime.utcnow() self._manhole_service.status = 'started' started_topic = '{}.on_manhole_started'.format(self._uri_prefix) started_info = self._manhole_service.marshal() self.publish(started_topic, started_info, options=PublishOptions(exclude=details.caller)) returnValue(started_info) @wamp.register(None) @inlineCallbacks def stop_manhole(self, details=None): """ Stop the Manhole service running in this process. This procedure is registered under * ``crossbar.node.<node_id>.worker.<worker_id>.stop_manhole`` for native workers and under * ``crossbar.node.<node_id>.controller.stop_manhole`` for node controllers When no Manhole service is currently running within this process, or the Manhole service is already shutting down, a ``crossbar.error.not_started`` WAMP error is raised. The procedure will publish an event when the service **is stopping** to * ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_stopping`` for native workers and * ``crossbar.node.<node_id>.controller.on_manhole_stopping`` for node controllers and will publish an event when the service **has stopped** to * ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_stopped`` for native workers and * ``crossbar.node.<node_id>.controller.on_manhole_stopped`` for node controllers """ self.log.debug("{cls}.stop_manhole", cls=self.__class__.__name__) if not self._manhole_service or self._manhole_service.status != 'started': emsg = "Cannot stop manhole: not running (or already shutting down)" raise ApplicationError(u"crossbar.error.not_started", emsg) self._manhole_service.status = 'stopping' stopping_topic = u'{}.on_manhole_stopping'.format(self._uri_prefix) stopping_info = None # the caller gets a progressive result .. if details.progress: details.progress(stopping_info) # .. while all others get an event self.publish(stopping_topic, stopping_info, options=PublishOptions(exclude=details.caller)) try: yield self._manhole_service.port.stopListening() except Exception as e: self.log.warn("error while stop listening on endpoint: {error}", error=e) self._manhole_service = None stopped_topic = u'{}.on_manhole_stopped'.format(self._uri_prefix) stopped_info = None self.publish(stopped_topic, stopped_info, options=PublishOptions(exclude=details.caller)) returnValue(stopped_info) @wamp.register(None) def get_manhole(self, details=None): """ Get current manhole service information. :returns: dict -- A dict with service information or `None` if the service is not running. """ self.log.debug("{cls}.get_manhole", cls=self.__class__.__name__) if not _HAS_MANHOLE: emsg = "Could not start manhole: required packages are missing ({})".format( _MANHOLE_MISSING_REASON) self.log.error(emsg) raise ApplicationError(u"crossbar.error.feature_unavailable", emsg) if not self._manhole_service: return None else: return self._manhole_service.marshal() @wamp.register(None) def utcnow(self, details=None): """ Return current time as determined from within this process. **Usage:** This procedure is registered under * ``crossbar.node.<node_id>.worker.<worker_id>.utcnow`` for native workers and under * ``crossbar.node.<node_id>.controller.utcnow`` for node controllers :returns: Current time (UTC) in UTC ISO 8601 format. :rtype: unicode """ self.log.debug("{cls}.utcnow", cls=self.__class__.__name__) return utcnow() @wamp.register(None) def started(self, details=None): """ Return start time of this process. **Usage:** This procedure is registered under * ``crossbar.node.<node_id>.worker.<worker_id>.started`` for native workers and under * ``crossbar.node.<node_id>.controller.started`` for node controllers :returns: Start time (UTC) in UTC ISO 8601 format. :rtype: unicode """ self.log.debug("{cls}.started", cls=self.__class__.__name__) return utcstr(self._started) @wamp.register(None) def uptime(self, details=None): """ Return uptime of this process. **Usage:** This procedure is registered under * ``crossbar.node.<node_id>.worker.<worker_id>.uptime`` for native workers and under * ``crossbar.node.<node_id>.controller.uptime`` for node controllers :returns: Uptime in seconds. :rtype: float """ self.log.debug("{cls}.uptime", cls=self.__class__.__name__) now = datetime.utcnow() return (now - self._started).total_seconds()
class NativeProcessSession(ApplicationSession): """ A native Crossbar.io process (currently: controller, router or container). """ def onConnect(self, do_join=True): """ """ if not hasattr(self, 'debug'): self.debug = self.config.extra.debug if not hasattr(self, 'cbdir'): self.cbdir = self.config.extra.cbdir if not hasattr(self, '_uri_prefix'): self._uri_prefix = 'crossbar.node.{}'.format( self.config.extra.node) if self.debug: log.msg("Session connected to management router") self._started = datetime.utcnow() # see: BaseSession self.include_traceback = False self.debug_app = False self._manhole_service = None if _HAS_PSUTIL: self._pinfo = ProcessInfo() self._pinfo_monitor = None self._pinfo_monitor_seq = 0 else: self._pinfo = None self._pinfo_monitor = None self._pinfo_monitor_seq = None log.msg("Warning: process utilities not available") if do_join: self.join(self.config.realm) @inlineCallbacks def onJoin(self, details): """ Called when process has joined the node's management realm. """ procs = [ 'start_manhole', 'stop_manhole', 'get_manhole', 'trigger_gc', 'utcnow', 'started', 'uptime', 'get_process_info', 'get_process_stats', 'set_process_stats_monitoring' ] dl = [] for proc in procs: uri = '{}.{}'.format(self._uri_prefix, proc) if self.debug: log.msg("Registering procedure '{}'".format(uri)) dl.append( self.register(getattr(self, proc), uri, options=RegisterOptions(details_arg='details'))) regs = yield DeferredList(dl) if self.debug: log.msg("{} registered {} procedures".format( self.__class__.__name__, len(regs))) def get_process_info(self, details=None): """ Get process information (open files, sockets, ...). :returns: dict -- Dictionary with process information. """ if self.debug: log.msg("{}.get_process_info".format(self.__class__.__name__)) if self._pinfo: return self._pinfo.get_info() else: emsg = "ERROR: could not retrieve process statistics - required packages not installed" raise ApplicationError("crossbar.error.feature_unavailable", emsg) def get_process_stats(self, details=None): """ Get process statistics (CPU, memory, I/O). :returns: dict -- Dictionary with process statistics. """ if self.debug: log.msg("{}.get_process_stats".format(self.__class__.__name__)) if self._pinfo: return self._pinfo.get_stats() else: emsg = "ERROR: could not retrieve process statistics - required packages not installed" raise ApplicationError("crossbar.error.feature_unavailable", emsg) def set_process_stats_monitoring(self, interval, details=None): """ Enable/disable periodic publication of process statistics. :param interval: The monitoring interval in seconds. Set to 0 to disable monitoring. :type interval: float """ if self.debug: log.msg( "{}.set_process_stats_monitoring".format( self.__class__.__name__), interval) if self._pinfo: stats_monitor_set_topic = '{}.on_process_stats_monitoring_set'.format( self._uri_prefix) # stop and remove any existing monitor if self._pinfo_monitor: self._pinfo_monitor.stop() self._pinfo_monitor = None self.publish(stats_monitor_set_topic, 0, options=PublishOptions(exclude=[details.caller])) # possibly start a new monitor if interval > 0: stats_topic = '{}.on_process_stats'.format(self._uri_prefix) def publish_stats(): stats = self._pinfo.get_stats() self._pinfo_monitor_seq += 1 stats['seq'] = self._pinfo_monitor_seq self.publish(stats_topic, stats) self._pinfo_monitor = LoopingCall(publish_stats) self._pinfo_monitor.start(interval) self.publish(stats_monitor_set_topic, interval, options=PublishOptions(exclude=[details.caller])) else: emsg = "ERROR: cannot setup process statistics monitor - required packages not installed" raise ApplicationError("crossbar.error.feature_unavailable", emsg) def trigger_gc(self, details=None): """ Triggers a garbage collection. :returns: float -- Time consumed for GC in ms. """ if self.debug: log.msg("{}.trigger_gc".format(self.__class__.__name__)) started = rtime() gc.collect() return 1000. * (rtime() - started) @inlineCallbacks def start_manhole(self, config, details=None): """ Start a manhole (SSH) within this worker. :param config: Manhole configuration. :type config: obj """ if self.debug: log.msg("{}.start_manhole".format(self.__class__.__name__), config) if not _HAS_MANHOLE: emsg = "ERROR: could not start manhole - required packages are missing ({})".format( _MANHOLE_MISSING_REASON) log.msg(emsg) raise ApplicationError("crossbar.error.feature_unavailable", emsg) if self._manhole_service: emsg = "ERROR: could not start manhole - already running (or starting)" log.msg(emsg) raise ApplicationError("crossbar.error.already_started", emsg) try: checkconfig.check_manhole(config) except Exception as e: emsg = "ERROR: could not start manhole - invalid configuration ({})".format( e) log.msg(emsg) raise ApplicationError('crossbar.error.invalid_configuration', emsg) # setup user authentication # checker = checkers.InMemoryUsernamePasswordDatabaseDontUse() for user in config['users']: checker.addUser(user['user'], user['password']) # setup manhole namespace # namespace = {'session': self} class PatchedTerminalSession(TerminalSession): # get rid of # exceptions.AttributeError: TerminalSession instance has no attribute 'windowChanged' def windowChanged(self, winSize): pass rlm = TerminalRealm() rlm.sessionFactory = PatchedTerminalSession # monkey patch rlm.chainedProtocolFactory.protocolFactory = lambda _: ColoredManhole( namespace) ptl = portal.Portal(rlm, [checker]) factory = ConchFactory(ptl) factory.noisy = False self._manhole_service = ManholeService(config, details.caller) starting_topic = '{}.on_manhole_starting'.format(self._uri_prefix) starting_info = self._manhole_service.marshal() # the caller gets a progressive result .. if details.progress: details.progress(starting_info) # .. while all others get an event self.publish(starting_topic, starting_info, options=PublishOptions(exclude=[details.caller])) try: self._manhole_service.port = yield create_listening_port_from_config( config['endpoint'], factory, self.cbdir, reactor) except Exception as e: self._manhole_service = None emsg = "ERROR: manhole service endpoint cannot listen - {}".format( e) log.msg(emsg) raise ApplicationError("crossbar.error.cannot_listen", emsg) # alright, manhole has started self._manhole_service.started = datetime.utcnow() self._manhole_service.status = 'started' started_topic = '{}.on_manhole_started'.format(self._uri_prefix) started_info = self._manhole_service.marshal() self.publish(started_topic, started_info, options=PublishOptions(exclude=[details.caller])) returnValue(started_info) @inlineCallbacks def stop_manhole(self, details=None): """ Stop Manhole. """ if self.debug: log.msg("{}.stop_manhole".format(self.__class__.__name__)) if not _HAS_MANHOLE: emsg = "ERROR: could not start manhole - required packages are missing ({})".format( _MANHOLE_MISSING_REASON) log.msg(emsg) raise ApplicationError("crossbar.error.feature_unavailable", emsg) if not self._manhole_service or self._manhole_service.status != 'started': emsg = "ERROR: cannot stop manhole - not running (or already shutting down)" raise ApplicationError("crossbar.error.not_started", emsg) self._manhole_service.status = 'stopping' stopping_topic = '{}.on_manhole_stopping'.format(self._uri_prefix) stopping_info = None # the caller gets a progressive result .. if details.progress: details.progress(stopping_info) # .. while all others get an event self.publish(stopping_topic, stopping_info, options=PublishOptions(exclude=[details.caller])) try: yield self._manhole_service.port.stopListening() except Exception as e: raise Exception( "INTERNAL ERROR: don't know how to handle a failed called to stopListening() - {}" .format(e)) self._manhole_service = None stopped_topic = '{}.on_manhole_stopped'.format(self._uri_prefix) stopped_info = None self.publish(stopped_topic, stopped_info, options=PublishOptions(exclude=[details.caller])) returnValue(stopped_info) def get_manhole(self, details=None): """ Get current manhole service information. :returns: dict -- A dict with service information or `None` if the service is not running. """ if self.debug: log.msg("{}.get_manhole".format(self.__class__.__name__)) if not _HAS_MANHOLE: emsg = "ERROR: could not start manhole - required packages are missing ({})".format( _MANHOLE_MISSING_REASON) log.msg(emsg) raise ApplicationError("crossbar.error.feature_unavailable", emsg) if not self._manhole_service: return None else: return self._manhole_service.marshal() def utcnow(self, details=None): """ Return current time as determined from within this process. :returns str -- Current time (UTC) in UTC ISO 8601 format. """ if self.debug: log.msg("{}.utcnow".format(self.__class__.__name__)) return utcnow() def started(self, details=None): """ Return start time of this process. :returns str -- Start time (UTC) in UTC ISO 8601 format. """ if self.debug: log.msg("{}.started".format(self.__class__.__name__)) return utcstr(self._started) def uptime(self, details=None): """ Uptime of this process. :returns float -- Uptime in seconds. """ if self.debug: log.msg("{}.uptime".format(self.__class__.__name__)) now = datetime.utcnow() return (now - self._started).total_seconds()
class AppSession(ApplicationSession): @inlineCallbacks def onJoin(self, details): self._db = Database(dbpath='../results') self._schema = Schema.attach(self._db) self._pinfo = ProcessInfo() self._logname = self.config.extra['logname'] self._period = self.config.extra.get('period', 10.) self._running = True batch_id = uuid.uuid4() self._last_stats = None self._stats_loop = LoopingCall(self._stats, batch_id) self._stats_loop.start(self._period) dl = [] for i in range(8): d = self._loop(batch_id, i) dl.append(d) d = gatherResults(dl) try: yield d except TransportLost: pass def onLeave(self, details): self._running = False self._stats_loop.stop() def _stats(self, batch_id): stats = self._pinfo.get_stats() if self._last_stats: batch_duration = (stats['time'] - self._last_stats['time']) / 10**9 ctx = round((stats['voluntary'] - self._last_stats['voluntary']) / batch_duration, 0) self.log.info( '{logprefix}: {user} user, {system} system, {mem_percent} mem_percent, {ctx} ctx', logprefix=hl('LOAD {}.*'.format(self._logname), color='magenta', bold=True), logname=self._logname, user=stats['user'], system=stats['system'], mem_percent=round(stats['mem_percent'], 1), ctx=ctx) self._last_stats = stats @inlineCallbacks def _loop(self, batch_id, index): prefix = self.config.extra['prefix'] while self._running: rtts = [] self.log.debug( '{logprefix} is starting new period ..', logprefix=hl(' {}.{}'.format(self._logname, index), color='magenta', bold=True), ) batch_started = time_ns() while float(time_ns() - batch_started) / 10**9 < self._period: ts_req = time_ns() res = yield self.call('{}.echo'.format(prefix), ts_req) ts_res = time_ns() assert res == ts_req rtt = ts_res - ts_req rtts.append(rtt) batch_duration = float(time_ns() - batch_started) / 10**9 rtts = sorted(rtts) sr = WampStatsRecord() sr.worker = int(self._logname.split('.')[1]) sr.loop = index sr.count = len(rtts) sr.calls_per_sec = int(round(sr.count / batch_duration, 0)) # all times here are in microseconds: sr.avg_rtt = int( round(1000000. * batch_duration / float(sr.count), 0)) sr.max_rtt = int(round(rtts[-1] / 1000, 0)) sr.q50_rtt = int(round(rtts[int(sr.count / 2.)] / 1000, 0)) sr.q99_rtt = int(round(rtts[int(-(sr.count / 100.))] / 1000, 0)) sr.q995_rtt = int(round(rtts[int(-(sr.count / 995.))] / 1000, 0)) try: with self._db.begin(write=True) as txn: self._schema.wamp_stats[txn, ( batch_id, np.datetime64(batch_started, 'ns'))] = sr except: self.log.failure() return self.leave() self.log.info( "{logprefix}: {count} calls, {calls_per_sec}, round-trip time (microseconds): q50 {q50_rtt}, avg {avg_rtt}, {q99_rtt}, q995 {q995_rtt}, max {max_rtt}" .format(logprefix=hl('WAMP {}.{}'.format(self._logname, index), color='magenta', bold=True), count=sr.count, calls_per_sec=hl('{} calls/second'.format( sr.calls_per_sec), bold=True), q50_rtt=sr.q50_rtt, avg_rtt=sr.avg_rtt, q99_rtt=hl('q99 {}'.format(sr.q99_rtt), bold=True), q995_rtt=sr.q995_rtt, max_rtt=sr.max_rtt))
class HATestClientSession(ApplicationSession): TEST_TOPIC = 'com.example.test1.binary' TEST_PROC = 'com.example.proc1.binary' log = txaio.make_logger() def proc1(self, logname, url, loop, counter, payload, details=None): fingerprint = hashlib.sha256(payload).digest()[:6] self.log.info( '{logprefix}: INVOCATION received on pid={pid} from {sender} -> loop={loop}, counter={counter}, len={payload_len}, fp={fp}, caller={caller}, caller_authid={caller_authid}, caller_authrole={caller_authrole}, forward_for={forward_for}', logprefix=hl('WAMP {}:{}'.format(self._url, self._logname), color='blue', bold=True), pid=hl(self._pid, color='blue', bold=True), sender=hl('{}:{}'.format(url, logname), color='blue', bold=True), loop=loop, counter=counter, procedure=details.procedure, payload_len=len(payload), fp=hl(binascii.b2a_hex(fingerprint).decode(), color='blue', bold=True), caller=hl(details.caller, color='blue', bold=True), caller_authid=hl(details.caller_authid, color='blue', bold=True), caller_authrole=hl(details.caller_authrole, color='blue', bold=True), forward_for=details.forward_for) fingerprint = hashlib.sha256(payload).digest()[:6] self._received_calls_cnt += 1 self._received_calls_bytes += len(payload) if details.forward_for: self._received_calls_ff_cnt += 1 return fingerprint, self._pid, self._logname, self._url def on_event1(self, logname, url, loop, counter, payload, details=None): fingerprint = hashlib.sha256(payload).digest()[:6] self.log.debug( '{logprefix}: EVENT received from {sender} -> loop={loop}, counter={counter}, len={payload_len}, fp={fp}, publisher={publisher}, publisher_authid={publisher_authid}, publisher_authole={publisher_authrole}, forward_for={forward_for}', logprefix=hl('WAMP {}:{}'.format(self._url, self._logname), color='blue', bold=True), sender=hl('{}:{}'.format(url, logname), color='blue', bold=True), loop=loop, counter=counter, topic=details.topic, payload_len=len(payload), fp=hl(binascii.b2a_hex(fingerprint).decode(), color='blue', bold=True), publisher=hl(details.publisher, color='blue', bold=True), publisher_authid=hl(details.publisher_authid, color='blue', bold=True), publisher_authrole=hl(details.publisher_authrole, color='blue', bold=True), forward_for=details.forward_for) self._received_cnt += 1 self._received_bytes += len(payload) if details.forward_for: self._received_ff_cnt += 1 @inlineCallbacks def onJoin(self, details): self._pid = os.getpid() # benchmark parametrization: self._period = self.config.extra.get('period', 10) self._loops = self.config.extra.get('loops', 1) self._rate = self.config.extra.get('rate', 1) self._stride = self.config.extra.get('stride', 1) self._size = self.config.extra.get('size', 256) self._logname = self.config.extra.get('logname', None) self._url = self.config.extra.get('url', None) self._silent = self.config.extra.get('silent', False) self._batch_id = uuid.uuid4() self._running = True self._pinfo = ProcessInfo() # run-time session statistics for EVENTs self._received_cnt = 0 self._received_bytes = 0 self._received_ff_cnt = 0 self._published_cnt = 0 self._published_bytes = 0 # run-time session statistics for CALLs self._received_calls_cnt = 0 self._received_calls_bytes = 0 self._received_calls_ff_cnt = 0 self._calls_cnt = 0 self._calls_bytes = 0 self.log.info('{logname} connected [batch="{batch_id}"]: {details}', logname=self._logname, batch_id=hl(self._batch_id), details=details) self._last_stats = None self._stats_loop = None if self._silent: self._stats(self._batch_id, self.log.info) stats_period = 10. if stats_period: self._stats_loop = LoopingCall(self._stats, self._batch_id, self.log.info) self._stats_loop.start(stats_period) yield self.subscribe(self.on_event1, HATestClientSession.TEST_TOPIC, options=SubscribeOptions(match='exact', details=True)) for i in range(self._loops): self._sender_loop('{}.{}'.format(self._logname, i)) self.log.info( '{logname} ready [period={period}, loops={loops}, rate={rate}, stride={stride}, size={size}]', logname=self._logname, period=self._period, loops=self._loops, rate=self._rate, stride=self._stride, size=self._size) def onLeave(self, details): self.log.info('{logname} leaving: reason="{reason}"', logname=self._logname, reason=details.reason) self._running = False if self._silent: self._stats(self._batch_id, self.log.warn) if self._stats_loop: self._stats_loop.stop() self._stats_loop = None @inlineCallbacks def _sender_loop(self, loopname, enable_publish=True, enable_call=True): loop = 0 while self._running: started = time_ns() dl = [] for counter in range(self._stride): payload = os.urandom(self._size) fingerprint = hashlib.sha256(payload).digest()[:6] if enable_publish: d = self.publish(HATestClientSession.TEST_TOPIC, self._logname, self._url, loop, counter, payload, options=PublishOptions(acknowledge=True, exclude_me=False)) dl.append(d) self._published_cnt += 1 self._published_bytes += len(payload) self.log.debug( '{logprefix}: EVENT sent from session={session}, authid={authid}, authrole={authrole} -> loop={loop}, counter={counter}, len={payload_len}, fp={fp}', logprefix=hl('WAMP {}:{}'.format( self._url, self._logname), color='green', bold=True), session=hl(self._session_id, color='green', bold=True), authid=hl(self._authid, color='green', bold=True), authrole=hl(self._authrole, color='green', bold=True), loop=loop, counter=counter, topic=HATestClientSession.TEST_TOPIC, payload_len=len(payload), fp=hl(binascii.b2a_hex(fingerprint).decode(), color='green', bold=True)) if enable_call: for uri in [ 'node{}.container1.proc1'.format(i + 1) for i in range(4) ]: d = self.call(uri, self._logname, self._url, loop, counter, payload, options=CallOptions(details=True)) def check_result(result, uri): print('-' * 100, result) _fingerprint, _pid, _logname, _url = result.results[ 0] self.log.info( '{logprefix}: CALL RESULT for {uri} received from pid={pid}, logname={logname}, url={url}, callee={callee}, callee_authid={callee_authid}, callee_authrole={callee_authrole}, forward_for={forward_for}, fp={fp} => fp_equal={fp_equal}', logprefix=hl('WAMP {}:{}'.format( self._url, self._logname), color='green', bold=True), pid=hl(_pid, color='green', bold=True), logname=_logname, url=_url, fp=hl(binascii.b2a_hex(fingerprint).decode(), color='green', bold=True), fp_equal=(_fingerprint == fingerprint), uri=hl(uri, color='yellow', bold=True), callee=result.callee, callee_authid=result.callee_authid, callee_authrole=result.callee_authrole, forward_for=result.forward_for) assert _fingerprint == fingerprint def error(err): print(err) d.addCallbacks(check_result, error, (uri, )) dl.append(d) self._calls_cnt += 1 self._calls_bytes += len(payload) self.log.info( '{logprefix}: CALL issued to {uri} from pid={pid}, session={session}, authid={authid}, authrole={authrole} -> loop={loop}, counter={counter}, len={payload_len}, fp={fp}', logprefix=hl('WAMP {}:{}'.format( self._url, self._logname), color='green', bold=True), pid=hl(self._pid, color='green', bold=True), session=hl(self._session_id, color='green', bold=True), authid=hl(self._authid, color='green', bold=True), authrole=hl(self._authrole, color='green', bold=True), uri=hl(uri, color='yellow', bold=True), loop=loop, counter=counter, procedure=HATestClientSession.TEST_PROC, payload_len=len(payload), fp=hl(binascii.b2a_hex(fingerprint).decode(), color='green', bold=True)) d = gatherResults(dl) try: yield d except TransportLost: self.log.error('Transport lost!') self.leave() return duration = (time_ns() - started) / 10**9 sleep_secs = (1 / float(self._rate)) - duration if sleep_secs > 0: yield sleep(sleep_secs) loop += 1 def _stats(self, batch_id, log): stats = self._pinfo.get_stats() if self._last_stats: batch_duration = (stats['time'] - self._last_stats['time']) / 10**9 ctx = round((stats['voluntary'] - self._last_stats['voluntary']) / batch_duration, 0) log('{logprefix}: {user} user, {system} system, {mem_percent} mem_percent, {ctx} ctx', logprefix=hl('LOAD', color='white', bold=True), user=stats['user'], system=stats['system'], mem_percent=round(stats['mem_percent'], 1), ctx=ctx) events_received_per_sec = int( round(self._received_cnt / batch_duration)) bytes_received_per_sec = int( round(self._received_bytes / batch_duration)) events_published_per_sec = int( round(self._published_cnt / batch_duration)) bytes_published_per_sec = int( round(self._published_bytes / batch_duration)) log('{logprefix}: {events_received} EVENTs received ({events_received_ff} forwarded), {events_received_per_sec}, {events_published} events published, {events_published_per_sec} events/second', logprefix=hl('WAMP {}.*'.format(self._logname), color='white', bold=True), events_received=self._received_cnt, events_received_ff=self._received_ff_cnt, events_received_per_sec=hl( '{} events/second'.format(events_received_per_sec), color='white', bold=True), bytes_received_per_sec=bytes_received_per_sec, events_published=self._published_cnt, events_published_per_sec=events_published_per_sec, bytes_published_per_sec=bytes_published_per_sec) calls_received_per_sec = int( round(self._received_calls_cnt / batch_duration)) call_bytes_received_per_sec = int( round(self._received_calls_bytes / batch_duration)) calls_issued_per_sec = int(round(self._calls_cnt / batch_duration)) call_bytes_issued_per_sec = int( round(self._calls_bytes / batch_duration)) log('{logprefix}: {calls_received} INVOCATIONs received ({calls_received_ff} forwarded), {calls_received_per_sec}, {calls_issued} calls issued, {calls_issued_per_sec} calls/second', logprefix=hl('WAMP {}.*'.format(self._logname), color='white', bold=True), calls_received=self._received_calls_cnt, calls_received_ff=self._received_calls_ff_cnt, calls_received_per_sec=hl( '{} calls/second'.format(calls_received_per_sec), color='white', bold=True), call_bytes_received_per_sec=call_bytes_received_per_sec, calls_issued=self._calls_cnt, calls_issued_per_sec=calls_issued_per_sec, call_bytes_issued_per_sec=call_bytes_issued_per_sec) self._received_cnt = 0 self._received_bytes = 0 self._received_ff_cnt = 0 self._published_cnt = 0 self._published_bytes = 0 self._received_calls_cnt = 0 self._received_calls_bytes = 0 self._received_calls_ff_cnt = 0 self._calls_cnt = 0 self._calls_bytes = 0 self._last_stats = stats
def onJoin(self, details): self._pid = os.getpid() # benchmark parametrization: self._period = self.config.extra.get('period', 10) self._loops = self.config.extra.get('loops', 1) self._rate = self.config.extra.get('rate', 1) self._stride = self.config.extra.get('stride', 1) self._size = self.config.extra.get('size', 256) self._logname = self.config.extra.get('logname', None) self._url = self.config.extra.get('url', None) self._silent = self.config.extra.get('silent', False) self._batch_id = uuid.uuid4() self._running = True self._pinfo = ProcessInfo() # run-time session statistics for EVENTs self._received_cnt = 0 self._received_bytes = 0 self._received_ff_cnt = 0 self._published_cnt = 0 self._published_bytes = 0 # run-time session statistics for CALLs self._received_calls_cnt = 0 self._received_calls_bytes = 0 self._received_calls_ff_cnt = 0 self._calls_cnt = 0 self._calls_bytes = 0 self.log.info('{logname} connected [batch="{batch_id}"]: {details}', logname=self._logname, batch_id=hl(self._batch_id), details=details) self._last_stats = None self._stats_loop = None if self._silent: self._stats(self._batch_id, self.log.info) stats_period = 10. if stats_period: self._stats_loop = LoopingCall(self._stats, self._batch_id, self.log.info) self._stats_loop.start(stats_period) yield self.subscribe(self.on_event1, HATestClientSession.TEST_TOPIC, options=SubscribeOptions(match='exact', details=True)) for i in range(self._loops): self._sender_loop('{}.{}'.format(self._logname, i)) self.log.info( '{logname} ready [period={period}, loops={loops}, rate={rate}, stride={stride}, size={size}]', logname=self._logname, period=self._period, loops=self._loops, rate=self._rate, stride=self._stride, size=self._size)
class NativeProcessSession(ApplicationSession): """ A native Crossbar.io process (currently: controller, router or container). """ def onConnect(self, do_join = True): """ """ if not hasattr(self, 'debug'): self.debug = self.config.extra.debug if not hasattr(self, 'cbdir'): self.cbdir = self.config.extra.cbdir if not hasattr(self, '_uri_prefix'): self._uri_prefix = 'crossbar.node.{}'.format(self.config.extra.node) if self.debug: log.msg("Session connected to management router") self._started = datetime.utcnow() ## see: BaseSession self.include_traceback = False self.debug_app = False self._manhole_service = None if _HAS_PSUTIL: self._pinfo = ProcessInfo() self._pinfo_monitor = None self._pinfo_monitor_seq = 0 else: self._pinfo = None self._pinfo_monitor = None self._pinfo_monitor_seq = None log.msg("Warning: process utilities not available") if do_join: self.join(self.config.realm) @inlineCallbacks def onJoin(self, details): """ Called when process has joined the node's management realm. """ procs = [ 'start_manhole', 'stop_manhole', 'get_manhole', 'trigger_gc', 'utcnow', 'started', 'uptime', 'get_process_info', 'get_process_stats', 'set_process_stats_monitoring' ] dl = [] for proc in procs: uri = '{}.{}'.format(self._uri_prefix, proc) if self.debug: log.msg("Registering procedure '{}'".format(uri)) dl.append(self.register(getattr(self, proc), uri, options = RegisterOptions(details_arg = 'details', discloseCaller = True))) regs = yield DeferredList(dl) if self.debug: log.msg("{} registered {} procedures".format(self.__class__.__name__, len(regs))) def get_process_info(self, details = None): """ Get process information (open files, sockets, ...). :returns: dict -- Dictionary with process information. """ if self.debug: log.msg("{}.get_process_info".format(self.__class__.__name__)) if self._pinfo: return self._pinfo.get_info() else: emsg = "ERROR: could not retrieve process statistics - required packages not installed" raise ApplicationError("crossbar.error.feature_unavailable", emsg) def get_process_stats(self, details = None): """ Get process statistics (CPU, memory, I/O). :returns: dict -- Dictionary with process statistics. """ if self.debug: log.msg("{}.get_process_stats".format(self.__class__.__name__)) if self._pinfo: return self._pinfo.get_stats() else: emsg = "ERROR: could not retrieve process statistics - required packages not installed" raise ApplicationError("crossbar.error.feature_unavailable", emsg) def set_process_stats_monitoring(self, interval, details = None): """ Enable/disable periodic publication of process statistics. :param interval: The monitoring interval in seconds. Set to 0 to disable monitoring. :type interval: float """ if self.debug: log.msg("{}.set_process_stats_monitoring".format(self.__class__.__name__), interval) if self._pinfo: stats_monitor_set_topic = '{}.on_process_stats_monitoring_set'.format(self._uri_prefix) ## stop and remove any existing monitor if self._pinfo_monitor: self._pinfo_monitor.stop() self._pinfo_monitor = None self.publish(stats_monitor_set_topic, 0, options = PublishOptions(exclude = [details.caller])) ## possibly start a new monitor if interval > 0: stats_topic = '{}.on_process_stats'.format(self._uri_prefix) def publish_stats(): stats = self._pinfo.get_stats() self._pinfo_monitor_seq += 1 stats['seq'] = self._pinfo_monitor_seq self.publish(stats_topic, stats) self._pinfo_monitor = LoopingCall(publish_stats) self._pinfo_monitor.start(interval) self.publish(stats_monitor_set_topic, interval, options = PublishOptions(exclude = [details.caller])) else: emsg = "ERROR: cannot setup process statistics monitor - required packages not installed" raise ApplicationError("crossbar.error.feature_unavailable", emsg) def trigger_gc(self, details = None): """ Triggers a garbage collection. :returns: float -- Time consumed for GC in ms. """ if self.debug: log.msg("{}.trigger_gc".format(self.__class__.__name__)) started = rtime() gc.collect() return 1000. * (rtime() - started) @inlineCallbacks def start_manhole(self, config, details = None): """ Start a manhole (SSH) within this worker. :param config: Manhole configuration. :type config: obj """ if self.debug: log.msg("{}.start_manhole".format(self.__class__.__name__), config) if not _HAS_MANHOLE: emsg = "ERROR: could not start manhole - required packages are missing ({})".format(_MANHOLE_MISSING_REASON) log.msg(emsg) raise ApplicationError("crossbar.error.feature_unavailable", emsg) if self._manhole_service: emsg = "ERROR: could not start manhole - already running (or starting)" log.msg(emsg) raise ApplicationError("crossbar.error.already_started", emsg) try: checkconfig.check_manhole(config) except Exception as e: emsg = "ERROR: could not start manhole - invalid configuration ({})".format(e) log.msg(emsg) raise ApplicationError('crossbar.error.invalid_configuration', emsg) ## setup user authentication ## checker = checkers.InMemoryUsernamePasswordDatabaseDontUse() for user in config['users']: checker.addUser(user['user'], user['password']) ## setup manhole namespace ## namespace = {'session': self} class PatchedTerminalSession(TerminalSession): ## get rid of ## exceptions.AttributeError: TerminalSession instance has no attribute 'windowChanged' def windowChanged(self, winSize): pass rlm = TerminalRealm() rlm.sessionFactory = PatchedTerminalSession # monkey patch rlm.chainedProtocolFactory.protocolFactory = lambda _: ColoredManhole(namespace) ptl = portal.Portal(rlm, [checker]) factory = ConchFactory(ptl) factory.noisy = False self._manhole_service = ManholeService(config, details.authid) starting_topic = '{}.on_manhole_starting'.format(self._uri_prefix) starting_info = self._manhole_service.marshal() ## the caller gets a progressive result .. if details.progress: details.progress(starting_info) ## .. while all others get an event self.publish(starting_topic, starting_info, options = PublishOptions(exclude = [details.caller])) try: self._manhole_service.port = yield create_listening_port_from_config(config['endpoint'], factory, self.cbdir, reactor) except Exception as e: self._manhole_service = None emsg = "ERROR: manhole service endpoint cannot listen - {}".format(e) log.msg(emsg) raise ApplicationError("crossbar.error.cannot_listen", emsg) ## alright, manhole has started self._manhole_service.started = datetime.utcnow() self._manhole_service.status = 'started' started_topic = '{}.on_manhole_started'.format(self._uri_prefix) started_info = self._manhole_service.marshal() self.publish(started_topic, started_info, options = PublishOptions(exclude = [details.caller])) returnValue(started_info) @inlineCallbacks def stop_manhole(self, details = None): """ Stop Manhole. """ if self.debug: log.msg("{}.stop_manhole".format(self.__class__.__name__)) if not _HAS_MANHOLE: emsg = "ERROR: could not start manhole - required packages are missing ({})".format(_MANHOLE_MISSING_REASON) log.msg(emsg) raise ApplicationError("crossbar.error.feature_unavailable", emsg) if not self._manhole_service or self._manhole_service.status != 'started': emsg = "ERROR: cannot stop manhole - not running (or already shutting down)" raise ApplicationError("crossbar.error.not_started", emsg) self._manhole_service.status = 'stopping' stopping_topic = '{}.on_manhole_stopping'.format(self._uri_prefix) stopping_info = None ## the caller gets a progressive result .. if details.progress: details.progress(stopping_info) ## .. while all others get an event self.publish(stopping_topic, stopping_info, options = PublishOptions(exclude = [details.caller])) try: yield self._manhole_service.port.stopListening() except Exception as e: raise Exception("INTERNAL ERROR: don't know how to handle a failed called to stopListening() - {}".format(e)) self._manhole_service = None stopped_topic = '{}.on_manhole_stopped'.format(self._uri_prefix) stopped_info = None self.publish(stopped_topic, stopped_info, options = PublishOptions(exclude = [details.caller])) returnValue(stopped_info) def get_manhole(self, details = None): """ Get current manhole service information. :returns: dict -- A dict with service information or `None` if the service is not running. """ if self.debug: log.msg("{}.get_manhole".format(self.__class__.__name__)) if not _HAS_MANHOLE: emsg = "ERROR: could not start manhole - required packages are missing ({})".format(_MANHOLE_MISSING_REASON) log.msg(emsg) raise ApplicationError("crossbar.error.feature_unavailable", emsg) if not self._manhole_service: return None else: return self._manhole_service.marshal() def utcnow(self, details = None): """ Return current time as determined from within this process. :returns str -- Current time (UTC) in UTC ISO 8601 format. """ if self.debug: log.msg("{}.utcnow".format(self.__class__.__name__)) return utcnow() def started(self, details = None): """ Return start time of this process. :returns str -- Start time (UTC) in UTC ISO 8601 format. """ if self.debug: log.msg("{}.started".format(self.__class__.__name__)) return utcstr(self._started) def uptime(self, details = None): """ Uptime of this process. :returns float -- Uptime in seconds. """ if self.debug: log.msg("{}.uptime".format(self.__class__.__name__)) now = datetime.utcnow() return (now - self._started).total_seconds()
class NativeProcessSession(ApplicationSession): """ A native Crossbar.io process (currently: controller, router or container). """ log = make_logger() def onConnect(self, do_join=True): """ """ if not hasattr(self, 'cbdir'): self.cbdir = self.config.extra.cbdir if not hasattr(self, '_uri_prefix'): self._uri_prefix = 'crossbar.node.{}'.format(self.config.extra.node) self._started = datetime.utcnow() # see: BaseSession self.include_traceback = False self.debug_app = False self._manhole_service = None if _HAS_PSUTIL: self._pinfo = ProcessInfo() self._pinfo_monitor = None self._pinfo_monitor_seq = 0 else: self._pinfo = None self._pinfo_monitor = None self._pinfo_monitor_seq = None self.log.info("Process utilities not available") self._connections = {} if do_join: self.join(self.config.realm) @inlineCallbacks def onJoin(self, details): """ Called when process has joined the node's management realm. """ procs = [ 'start_manhole', 'stop_manhole', 'get_manhole', 'start_connection', 'stop_connection', 'get_connections', 'trigger_gc', 'utcnow', 'started', 'uptime', 'get_process_info', 'get_process_stats', 'set_process_stats_monitoring' ] dl = [] for proc in procs: uri = '{}.{}'.format(self._uri_prefix, proc) self.log.debug("Registering procedure '{uri}'", uri=uri) dl.append(self.register(getattr(self, proc), uri, options=RegisterOptions(details_arg='details'))) regs = yield DeferredList(dl) self.log.debug("Registered {len_reg} procedures", len_reg=len(regs)) @inlineCallbacks def start_connection(self, id, config, details=None): """ Starts a connection in this process. :param id: The ID for the started connection. :type id: unicode :param config: Connection configuration. :type config: dict :param details: Caller details. :type details: instance of :class:`autobahn.wamp.types.CallDetails` :returns dict -- The connection. """ self.log.debug("start_connection: id={id}, config={config}", id=id, config=config) # prohibit starting a component twice # if id in self._connections: emsg = "cannot start connection: a connection with id={} is already started".format(id) self.log.warn(emsg) raise ApplicationError("crossbar.error.invalid_configuration", emsg) # check configuration # try: checkconfig.check_connection(config) except Exception as e: emsg = "invalid connection configuration ({})".format(e) self.log.warn(emsg) raise ApplicationError("crossbar.error.invalid_configuration", emsg) else: self.log.info("Starting {} in process.".format(config['type'])) if config['type'] == u'postgresql.connection': if _HAS_POSTGRESQL: connection = PostgreSQLConnection(id, config) else: emsg = "unable to start connection - required PostgreSQL driver package not installed" self.log.warn(emsg) raise ApplicationError("crossbar.error.feature_unavailable", emsg) else: # should not arrive here raise Exception("logic error") self._connections[id] = connection try: yield connection.start() self.log.info("Connection {connection_type} started '{connection_id}'", connection_id=id, connection_type=config['type']) except Exception as e: del self._connections[id] raise state = connection.marshal() self.publish(u'crossbar.node.process.on_connection_start', state) returnValue(state) @inlineCallbacks def stop_connection(self, id, details=None): """ Stop a connection currently running within this process. :param id: The ID of the connection to stop. :type id: unicode :param details: Caller details. :type details: instance of :class:`autobahn.wamp.types.CallDetails` :returns dict -- A dict with component start information. """ self.log.debug("stop_connection: id={id}", id=id) if id not in self._connections: raise ApplicationError('crossbar.error.no_such_object', 'no connection with ID {} running in this process'.format(id)) connection = self._connections[id] try: yield connection.stop() except Exception as e: self.log.warn('could not stop connection {id}: {error}', error=e) raise del self._connections[id] state = connection.marshal() self.publish(u'crossbar.node.process.on_connection_stop', state) returnValue(state) def get_connections(self, details=None): """ Get connections currently running within this processs. :param details: Caller details. :type details: instance of :class:`autobahn.wamp.types.CallDetails` :returns list -- List of connections. """ self.log.debug("get_connections") res = [] for c in self._connections.values(): res.append(c.marshal()) return res def get_process_info(self, details=None): """ Get process information (open files, sockets, ...). :returns: dict -- Dictionary with process information. """ self.log.debug("{cls}.get_process_info", cls=self.__class__.__name__) if self._pinfo: return self._pinfo.get_info() else: emsg = "ERROR: could not retrieve process statistics - required packages not installed" raise ApplicationError("crossbar.error.feature_unavailable", emsg) def get_process_stats(self, details=None): """ Get process statistics (CPU, memory, I/O). :returns: dict -- Dictionary with process statistics. """ self.log.debug("{cls}.get_process_stats", cls=self.__class__.__name__) if self._pinfo: return self._pinfo.get_stats() else: emsg = "ERROR: could not retrieve process statistics - required packages not installed" raise ApplicationError("crossbar.error.feature_unavailable", emsg) def set_process_stats_monitoring(self, interval, details=None): """ Enable/disable periodic publication of process statistics. :param interval: The monitoring interval in seconds. Set to 0 to disable monitoring. :type interval: float """ self.log.debug("{cls}.set_process_stats_monitoring(interval = {interval})", cls=self.__class__.__name__, interval=interval) if self._pinfo: stats_monitor_set_topic = '{}.on_process_stats_monitoring_set'.format(self._uri_prefix) # stop and remove any existing monitor if self._pinfo_monitor: self._pinfo_monitor.stop() self._pinfo_monitor = None self.publish(stats_monitor_set_topic, 0, options=PublishOptions(exclude=[details.caller])) # possibly start a new monitor if interval > 0: stats_topic = '{}.on_process_stats'.format(self._uri_prefix) def publish_stats(): stats = self._pinfo.get_stats() self._pinfo_monitor_seq += 1 stats['seq'] = self._pinfo_monitor_seq self.publish(stats_topic, stats) self._pinfo_monitor = LoopingCall(publish_stats) self._pinfo_monitor.start(interval) self.publish(stats_monitor_set_topic, interval, options=PublishOptions(exclude=[details.caller])) else: emsg = "ERROR: cannot setup process statistics monitor - required packages not installed" raise ApplicationError("crossbar.error.feature_unavailable", emsg) def trigger_gc(self, details=None): """ Triggers a garbage collection. :returns: float -- Time consumed for GC in ms. """ self.msg.debug("{cls}.trigger_gc", cls=self.__class__.__name__) started = rtime() gc.collect() return 1000. * (rtime() - started) @inlineCallbacks def start_manhole(self, config, details=None): """ Start a manhole (SSH) within this worker. :param config: Manhole configuration. :type config: obj """ self.log.debug("{cls}.start_manhole(config = {config})", cls=self.__class__.__name__, config=config) if not _HAS_MANHOLE: emsg = "ERROR: could not start manhole - required packages are missing ({})".format(_MANHOLE_MISSING_REASON) self.log.error(emsg) raise ApplicationError("crossbar.error.feature_unavailable", emsg) if self._manhole_service: emsg = "ERROR: could not start manhole - already running (or starting)" self.log.warn(emsg) raise ApplicationError("crossbar.error.already_started", emsg) try: checkconfig.check_manhole(config) except Exception as e: emsg = "ERROR: could not start manhole - invalid configuration ({})".format(e) self.log.error(emsg) raise ApplicationError('crossbar.error.invalid_configuration', emsg) # setup user authentication # checker = checkers.InMemoryUsernamePasswordDatabaseDontUse() for user in config['users']: checker.addUser(user['user'], user['password']) # setup manhole namespace # namespace = {'session': self} class PatchedTerminalSession(TerminalSession): # get rid of # exceptions.AttributeError: TerminalSession instance has no attribute 'windowChanged' def windowChanged(self, winSize): pass rlm = TerminalRealm() rlm.sessionFactory = PatchedTerminalSession # monkey patch rlm.chainedProtocolFactory.protocolFactory = lambda _: ColoredManhole(namespace) ptl = portal.Portal(rlm, [checker]) factory = ConchFactory(ptl) factory.noisy = False self._manhole_service = ManholeService(config, details.caller) starting_topic = '{}.on_manhole_starting'.format(self._uri_prefix) starting_info = self._manhole_service.marshal() # the caller gets a progressive result .. if details.progress: details.progress(starting_info) # .. while all others get an event self.publish(starting_topic, starting_info, options=PublishOptions(exclude=[details.caller])) try: self._manhole_service.port = yield create_listening_port_from_config(config['endpoint'], factory, self.cbdir, reactor) except Exception as e: self._manhole_service = None emsg = "ERROR: manhole service endpoint cannot listen - {}".format(e) self.log.error(emsg) raise ApplicationError("crossbar.error.cannot_listen", emsg) # alright, manhole has started self._manhole_service.started = datetime.utcnow() self._manhole_service.status = 'started' started_topic = '{}.on_manhole_started'.format(self._uri_prefix) started_info = self._manhole_service.marshal() self.publish(started_topic, started_info, options=PublishOptions(exclude=[details.caller])) returnValue(started_info) @inlineCallbacks def stop_manhole(self, details=None): """ Stop Manhole. """ self.log.debug("{cls}.stop_manhole", cls=self.__class__.__name__) if not _HAS_MANHOLE: emsg = "ERROR: could not start manhole - required packages are missing ({})".format(_MANHOLE_MISSING_REASON) self.log.error(emsg) raise ApplicationError("crossbar.error.feature_unavailable", emsg) if not self._manhole_service or self._manhole_service.status != 'started': emsg = "ERROR: cannot stop manhole - not running (or already shutting down)" raise ApplicationError("crossbar.error.not_started", emsg) self._manhole_service.status = 'stopping' stopping_topic = '{}.on_manhole_stopping'.format(self._uri_prefix) stopping_info = None # the caller gets a progressive result .. if details.progress: details.progress(stopping_info) # .. while all others get an event self.publish(stopping_topic, stopping_info, options=PublishOptions(exclude=[details.caller])) try: yield self._manhole_service.port.stopListening() except Exception as e: raise Exception("INTERNAL ERROR: don't know how to handle a failed called to stopListening() - {}".format(e)) self._manhole_service = None stopped_topic = '{}.on_manhole_stopped'.format(self._uri_prefix) stopped_info = None self.publish(stopped_topic, stopped_info, options=PublishOptions(exclude=[details.caller])) returnValue(stopped_info) def get_manhole(self, details=None): """ Get current manhole service information. :returns: dict -- A dict with service information or `None` if the service is not running. """ self.log.debug("{cls}.get_manhole", cls=self.__class__.__name__) if not _HAS_MANHOLE: emsg = "ERROR: could not start manhole - required packages are missing ({})".format(_MANHOLE_MISSING_REASON) self.log.error(emsg) raise ApplicationError("crossbar.error.feature_unavailable", emsg) if not self._manhole_service: return None else: return self._manhole_service.marshal() def utcnow(self, details=None): """ Return current time as determined from within this process. :returns str -- Current time (UTC) in UTC ISO 8601 format. """ self.log.debug("{cls}.utcnow", cls=self.__class__.__name__) return utcnow() def started(self, details=None): """ Return start time of this process. :returns str -- Start time (UTC) in UTC ISO 8601 format. """ self.log.debug("{cls}.started", cls=self.__class__.__name__) return utcstr(self._started) def uptime(self, details=None): """ Uptime of this process. :returns float -- Uptime in seconds. """ self.log.debug("{cls}.uptime", cls=self.__class__.__name__) now = datetime.utcnow() return (now - self._started).total_seconds()
class AppSession(ApplicationSession): @inlineCallbacks def onJoin(self, details): self._db = Database(dbpath='../results') self._schema = Schema.attach(self._db) self._pinfo = ProcessInfo() self._logname = self.config.extra['logname'] self._period = self.config.extra.get('period', 5.) self._running = True dl = [] for i in range(8): d = self._loop(i) dl.append(d) d = gatherResults(dl) try: yield d except TransportLost: pass def onLeave(self, details): self._running = False @inlineCallbacks def _loop(self, index): prefix = self.config.extra['prefix'] last = None while self._running: rtts = [] batch_started_str = utcnow() batch_started = time() while (time() - batch_started) < self._period: ts_req = time_ns() res = yield self.call('{}.echo'.format(prefix), ts_req) ts_res = time_ns() assert res == ts_req rtt = ts_res - ts_req rtts.append(rtt) stats = self._pinfo.get_stats() if last: batch_duration = (stats['time'] - last['time']) / 10 ** 9 ctx = round((stats['voluntary'] - last['voluntary']) / batch_duration, 0) self.log.info('{logname}: {cpu} cpu, {mem} mem, {ctx} ctx', logname=self._logname, cpu=round(stats['cpu_percent'], 1), mem=round(stats['mem_percent'], 1), ctx=ctx) rtts = sorted(rtts) sr = WampStatsRecord() sr.key = '{}#{}.{}'.format(batch_started_str, self._logname, index) sr.count = len(rtts) sr.calls_per_sec = int(round(sr.count / batch_duration, 0)) # all times here are in microseconds: sr.avg_rtt = round(1000000. * batch_duration / float(sr.count), 1) sr.max_rtt = round(rtts[-1] / 1000, 1) sr.q50_rtt = round(rtts[int(sr.count / 2.)] / 1000, 1) sr.q99_rtt = round(rtts[int(-(sr.count / 100.))] / 1000, 1) sr.q995_rtt = round(rtts[int(-(sr.count / 995.))] / 1000, 1) with self._db.begin(write=True) as txn: self._schema.wamp_stats[txn, sr.key] = sr print("{}: {} calls, {} calls/sec, RTT (us): q50 {}, avg {}, q99 {}, q995 {}, max {}".format(sr.key, sr.count, sr.calls_per_sec, sr.q50_rtt, sr.avg_rtt, sr.q99_rtt, sr.q995_rtt, sr.max_rtt)) last = stats