def onConnect(self, do_join=True):
"""
"""
if not hasattr(self, 'cbdir'):
self.cbdir = self.config.extra.cbdir
if not hasattr(self, '_uri_prefix'):
self._uri_prefix = 'crossbar.node.{}'.format(
self.config.extra.node)
self._started = datetime.utcnow()
# see: BaseSession
self.include_traceback = False
self._manhole_service = None
if _HAS_PSUTIL:
self._pinfo = ProcessInfo()
self._pinfo_monitor = None
self._pinfo_monitor_seq = 0
else:
self._pinfo = None
self._pinfo_monitor = None
self._pinfo_monitor_seq = None
self.log.info("Process utilities not available")
self._connections = {}
if do_join:
self.join(self.config.realm)
def onJoin(self, details):
self._db = Database(dbpath='../results')
self._schema = Schema.attach(self._db)
self._pinfo = ProcessInfo()
self._logname = self.config.extra['logname']
self._period = self.config.extra.get('period', 10.)
self._running = True
batch_id = uuid.uuid4()
self._last_stats = None
self._stats_loop = LoopingCall(self._stats, batch_id)
self._stats_loop.start(self._period)
dl = []
for i in range(8):
d = self._loop(batch_id, i)
dl.append(d)
d = gatherResults(dl)
try:
yield d
except TransportLost:
pass
def onJoin(self, details):
self._prefix = self.config.extra['prefix']
self._logname = self.config.extra['logname']
self._pinfo = ProcessInfo()
def echo(arg):
return arg
yield self.register(echo,
'{}.echo'.format(self._prefix),
options=RegisterOptions(invoke='roundrobin'))
self.log.info('{} ready!'.format(self._logname))
last = None
while True:
stats = self._pinfo.get_stats()
if last:
secs = (stats['time'] - last['time']) / 10**9
ctx = round((stats['voluntary'] - last['voluntary']) / secs, 0)
self.log.info('{logname}: {cpu} cpu, {mem} mem, {ctx} ctx',
logname=self._logname,
cpu=round(stats['cpu_percent'], 1),
mem=round(stats['mem_percent'], 1),
ctx=ctx)
last = stats
yield sleep(5)
class AppSession(ApplicationSession):
log = txaio.make_logger()
@inlineCallbacks
def onJoin(self, details):
self._prefix = self.config.extra['prefix']
self._logname = self.config.extra['logname']
self._pinfo = ProcessInfo()
def echo(arg):
return arg
yield self.register(echo, '{}.echo'.format(self._prefix), options=RegisterOptions(invoke='roundrobin'))
self.log.info('{} ready!'.format(self._logname))
last = None
while True:
stats = self._pinfo.get_stats()
if last:
secs = (stats['time'] - last['time']) / 10**9
ctx = round((stats['voluntary'] - last['voluntary']) / secs, 0)
self.log.info('{logname}: {cpu} cpu, {mem} mem, {ctx} ctx',
logname=self._logname,
cpu=round(stats['cpu_percent'], 1),
mem=round(stats['mem_percent'], 1),
ctx=ctx)
last = stats
yield sleep(5)
def onConnect(self, do_join=True):
"""
"""
if not hasattr(self, 'cbdir'):
self.cbdir = self.config.extra.cbdir
if not hasattr(self, '_uri_prefix'):
self._uri_prefix = 'crossbar.node.{}'.format(self.config.extra.node)
self._started = datetime.utcnow()
# see: BaseSession
self.include_traceback = False
self.debug_app = False
self._manhole_service = None
if _HAS_PSUTIL:
self._pinfo = ProcessInfo()
self._pinfo_monitor = None
self._pinfo_monitor_seq = 0
else:
self._pinfo = None
self._pinfo_monitor = None
self._pinfo_monitor_seq = None
self.log.info("Process utilities not available")
self._connections = {}
if do_join:
self.join(self.config.realm)
def onJoin(self, details):
self._prefix = self.config.extra['prefix']
self._logname = self.config.extra['logname']
self._count = 0
self._pinfo = ProcessInfo()
def echo(arg):
self._count += 1
return arg
yield self.register(echo,
'{}.echo'.format(self._prefix),
options=RegisterOptions(invoke='roundrobin'))
self.log.info('{} ready!'.format(self._logname))
last = None
while True:
stats = self._pinfo.get_stats()
if last:
secs = (stats['time'] - last['time']) / 10**9
calls_per_sec = int(round(self._count / secs, 0))
ctx = round((stats['voluntary'] - last['voluntary']) / secs, 0)
self.log.info(
'{logprefix}: {user} user, {system} system, {mem_percent} mem_percent, {ctx} ctx',
logprefix=hl('LOAD {}'.format(self._logname),
color='cyan',
bold=True),
user=stats['user'],
system=stats['system'],
mem_percent=round(stats['mem_percent'], 1),
ctx=ctx)
self.log.info(
'{logprefix}: {calls} calls, {calls_per_sec} calls/second',
logprefix=hl('WAMP {}'.format(self._logname),
color='cyan',
bold=True),
calls=self._count,
calls_per_sec=hl(calls_per_sec, bold=True))
self._count = 0
last = stats
yield sleep(10)
def onJoin(self, details):
self._db = Database(dbpath='../results')
self._schema = Schema.attach(self._db)
self._pinfo = ProcessInfo()
self._logname = self.config.extra['logname']
self._period = self.config.extra.get('period', 5.)
self._running = True
dl = []
for i in range(8):
d = self._loop(i)
dl.append(d)
d = gatherResults(dl)
try:
yield d
except TransportLost:
pass
def onConnect(self, do_join=True):
"""
"""
if not hasattr(self, 'debug'):
self.debug = self.config.extra.debug
if not hasattr(self, 'cbdir'):
self.cbdir = self.config.extra.cbdir
if not hasattr(self, '_uri_prefix'):
self._uri_prefix = 'crossbar.node.{}'.format(
self.config.extra.node)
if self.debug:
log.msg("Session connected to management router")
self._started = datetime.utcnow()
# see: BaseSession
self.include_traceback = False
self.debug_app = False
self._manhole_service = None
if _HAS_PSUTIL:
self._pinfo = ProcessInfo()
self._pinfo_monitor = None
self._pinfo_monitor_seq = 0
else:
self._pinfo = None
self._pinfo_monitor = None
self._pinfo_monitor_seq = None
log.msg("Warning: process utilities not available")
if do_join:
self.join(self.config.realm)
def on_worker_connected(self, proto):
"""
Called immediately after the worker process has been forked.
IMPORTANT: this slightly differs between native workers and guest workers!
"""
assert (self.status == u'starting')
assert (self.connected is None)
assert (self.proto is None)
assert (self.pid is None)
assert (self.pinfo is None)
self.status = u'connected'
self.connected = datetime.utcnow()
self.proto = proto
self.pid = proto.transport.pid
self.pinfo = ProcessInfo(self.pid)
def onJoin(self, details):
self._db = Database(dbpath='../results')
self._schema = Schema.attach(self._db)
self._pinfo = ProcessInfo()
self._logname = self.config.extra['logname']
self._period = self.config.extra.get('period', 5.)
self._running = True
dl = []
for i in range(8):
d = self._loop(i)
dl.append(d)
d = gatherResults(dl)
try:
yield d
except TransportLost:
pass
def on_worker_started(self, proto=None):
"""
Called after the worker process is connected to the node
router and registered all its management APIs there.
The worker is now ready for use!
"""
assert (self.status in [u'starting', u'connected'])
assert (self.started is None)
assert (self.proto is not None or proto is not None)
if not self.pid:
self.pid = proto.transport.pid
if not self.pinfo:
self.pinfo = ProcessInfo(self.pid)
assert (self.pid is not None)
assert (self.pinfo is not None)
self.status = u'started'
self.proto = self.proto or proto
self.started = datetime.utcnow()
class NativeProcessSession(ApplicationSession):
"""
A native Crossbar.io process (currently: controller, router or container).
"""
log = make_logger()
def __init__(self, config=None, reactor=None):
if not reactor:
from twisted.internet import reactor
self._reactor = reactor
self._reactor = reactor
super(ApplicationSession, self).__init__(config=config)
def onConnect(self, do_join=True):
"""
"""
if not hasattr(self, 'cbdir'):
self.cbdir = self.config.extra.cbdir
if not hasattr(self, '_uri_prefix'):
self._uri_prefix = 'crossbar.node.{}'.format(
self.config.extra.node)
self._started = datetime.utcnow()
# see: BaseSession
self.include_traceback = False
self._manhole_service = None
if _HAS_PSUTIL:
self._pinfo = ProcessInfo()
self._pinfo_monitor = None
self._pinfo_monitor_seq = 0
else:
self._pinfo = None
self._pinfo_monitor = None
self._pinfo_monitor_seq = None
self.log.info("Process utilities not available")
self._connections = {}
if do_join:
self.join(self.config.realm)
@inlineCallbacks
def onJoin(self, details):
"""
Called when process has joined the node's management realm.
"""
regs = yield self.register(
self,
prefix=u'{}.'.format(self._uri_prefix),
options=RegisterOptions(details_arg='details'),
)
self.log.info("Registered {len_reg} procedures", len_reg=len(regs))
for reg in regs:
if isinstance(reg, Failure):
self.log.error("Failed to register: {f}",
f=reg,
log_failure=reg)
else:
self.log.info(' {proc}', proc=reg.procedure)
returnValue(regs)
@inlineCallbacks
def start_connection(self, id, config, details=None):
"""
Starts a connection in this process.
:param id: The ID for the started connection.
:type id: unicode
:param config: Connection configuration.
:type config: dict
:param details: Caller details.
:type details: instance of :class:`autobahn.wamp.types.CallDetails`
:returns dict -- The connection.
"""
self.log.debug("start_connection: id={id}, config={config}",
id=id,
config=config)
# prohibit starting a component twice
#
if id in self._connections:
emsg = "cannot start connection: a connection with id={} is already started".format(
id)
self.log.warn(emsg)
raise ApplicationError(u"crossbar.error.invalid_configuration",
emsg)
# check configuration
#
try:
checkconfig.check_connection(config)
except Exception as e:
emsg = "invalid connection configuration ({})".format(e)
self.log.warn(emsg)
raise ApplicationError(u"crossbar.error.invalid_configuration",
emsg)
else:
self.log.info("Starting {ptype} in process.", ptype=config['type'])
if config['type'] == u'postgresql.connection':
if _HAS_POSTGRESQL:
connection = PostgreSQLConnection(id, config)
else:
emsg = "unable to start connection - required PostgreSQL driver package not installed"
self.log.warn(emsg)
raise ApplicationError(u"crossbar.error.feature_unavailable",
emsg)
else:
# should not arrive here
raise Exception("logic error")
self._connections[id] = connection
try:
yield connection.start()
self.log.info(
"Connection {connection_type} started '{connection_id}'",
connection_id=id,
connection_type=config['type'])
except Exception as e:
del self._connections[id]
raise
state = connection.marshal()
self.publish(u'crossbar.node.process.on_connection_start', state)
returnValue(state)
@inlineCallbacks
def stop_connection(self, id, details=None):
"""
Stop a connection currently running within this process.
:param id: The ID of the connection to stop.
:type id: unicode
:param details: Caller details.
:type details: instance of :class:`autobahn.wamp.types.CallDetails`
:returns dict -- A dict with component start information.
"""
self.log.debug("stop_connection: id={id}", id=id)
if id not in self._connections:
raise ApplicationError(
u'crossbar.error.no_such_object',
'no connection with ID {} running in this process'.format(id))
connection = self._connections[id]
try:
yield connection.stop()
except Exception as e:
self.log.warn('could not stop connection {id}: {error}', error=e)
raise
del self._connections[id]
state = connection.marshal()
self.publish(u'crossbar.node.process.on_connection_stop', state)
returnValue(state)
def get_connections(self, details=None):
"""
Get connections currently running within this processs.
:param details: Caller details.
:type details: instance of :class:`autobahn.wamp.types.CallDetails`
:returns list -- List of connections.
"""
self.log.debug("get_connections")
res = []
for c in self._connections.values():
res.append(c.marshal())
return res
@wamp.register(None)
def get_process_info(self, details=None):
"""
Get process information (open files, sockets, ...).
:returns: dict -- Dictionary with process information.
"""
self.log.debug("{cls}.get_process_info", cls=self.__class__.__name__)
if self._pinfo:
return self._pinfo.get_info()
else:
emsg = "Could not retrieve process statistics: required packages not installed"
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
@wamp.register(None)
def get_process_stats(self, details=None):
"""
Get process statistics (CPU, memory, I/O).
:returns: dict -- Dictionary with process statistics.
"""
self.log.debug("{cls}.get_process_stats", cls=self.__class__.__name__)
if self._pinfo:
return self._pinfo.get_stats()
else:
emsg = "Could not retrieve process statistics: required packages not installed"
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
@wamp.register(None)
def set_process_stats_monitoring(self, interval, details=None):
"""
Enable/disable periodic publication of process statistics.
:param interval: The monitoring interval in seconds. Set to 0 to disable monitoring.
:type interval: float
"""
self.log.debug(
"{cls}.set_process_stats_monitoring(interval = {interval})",
cls=self.__class__.__name__,
interval=interval)
if self._pinfo:
stats_monitor_set_topic = '{}.on_process_stats_monitoring_set'.format(
self._uri_prefix)
# stop and remove any existing monitor
if self._pinfo_monitor:
self._pinfo_monitor.stop()
self._pinfo_monitor = None
self.publish(stats_monitor_set_topic,
0,
options=PublishOptions(exclude=details.caller))
# possibly start a new monitor
if interval > 0:
stats_topic = '{}.on_process_stats'.format(self._uri_prefix)
def publish_stats():
stats = self._pinfo.get_stats()
self._pinfo_monitor_seq += 1
stats[u'seq'] = self._pinfo_monitor_seq
self.publish(stats_topic, stats)
self._pinfo_monitor = LoopingCall(publish_stats)
self._pinfo_monitor.start(interval)
self.publish(stats_monitor_set_topic,
interval,
options=PublishOptions(exclude=details.caller))
else:
emsg = "Cannot setup process statistics monitor: required packages not installed"
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
@wamp.register(None)
def trigger_gc(self, details=None):
"""
Manually trigger a garbage collection in this native process.
This procedure is registered under
``crossbar.node.<node_id>.worker.<worker_id>.trigger_gc``
for native workers and under
``crossbar.node.<node_id>.controller.trigger_gc``
for node controllers.
The procedure will publish an event when the garabage collection has finished to
``crossbar.node.<node_id>.worker.<worker_id>.on_gc_finished``
for native workers and
``crossbar.node.<node_id>.controller.on_gc_finished``
for node controllers:
.. code-block:: javascript
{
"requester": {
"session_id": 982734923,
"auth_id": "bob",
"auth_role": "admin"
},
"duration": 190
}
.. note:: The caller of this procedure will NOT receive the event.
:returns: Time (wall clock) consumed for garbage collection in ms.
:rtype: int
"""
self.log.debug("{cls}.trigger_gc", cls=self.__class__.__name__)
started = rtime()
# now trigger GC .. this is blocking!
gc.collect()
duration = int(round(1000. * (rtime() - started)))
on_gc_finished = u'{}.on_gc_finished'.format(self._uri_prefix)
self.publish(
on_gc_finished,
{
u'requester': {
u'session_id': details.caller,
# FIXME:
u'auth_id': None,
u'auth_role': None
},
u'duration': duration
},
options=PublishOptions(exclude=details.caller))
return duration
@wamp.register(None)
@inlineCallbacks
def start_manhole(self, config, details=None):
"""
Start a Manhole service within this process.
**Usage:**
This procedure is registered under
* ``crossbar.node.<node_id>.worker.<worker_id>.start_manhole`` - for native workers
* ``crossbar.node.<node_id>.controller.start_manhole`` - for node controllers
The procedure takes a Manhole service configuration which defines
a listening endpoint for the service and a list of users including
passwords, e.g.
.. code-block:: javascript
{
"endpoint": {
"type": "tcp",
"port": 6022
},
"users": [
{
"user": "******",
"password": "******"
}
]
}
**Errors:**
The procedure may raise the following errors:
* ``crossbar.error.invalid_configuration`` - the provided configuration is invalid
* ``crossbar.error.already_started`` - the Manhole service is already running (or starting)
* ``crossbar.error.feature_unavailable`` - the required support packages are not installed
**Events:**
The procedure will publish an event when the service **is starting** to
* ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_starting`` - for native workers
* ``crossbar.node.<node_id>.controller.on_manhole_starting`` - for node controllers
and publish an event when the service **has started** to
* ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_started`` - for native workers
* ``crossbar.node.<node_id>.controller.on_manhole_started`` - for node controllers
:param config: Manhole service configuration.
:type config: dict
"""
self.log.debug("{cls}.start_manhole(config = {config})",
cls=self.__class__.__name__,
config=config)
if not _HAS_MANHOLE:
emsg = "Could not start manhole: required packages are missing ({})".format(
_MANHOLE_MISSING_REASON)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
if self._manhole_service:
emsg = "Could not start manhole - already running (or starting)"
self.log.warn(emsg)
raise ApplicationError(u"crossbar.error.already_started", emsg)
try:
checkconfig.check_manhole(config)
except Exception as e:
emsg = "Could not start manhole: invalid configuration ({})".format(
e)
self.log.error(emsg)
raise ApplicationError(u'crossbar.error.invalid_configuration',
emsg)
# setup user authentication
#
checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()
for user in config['users']:
checker.addUser(user['user'], user['password'])
# setup manhole namespace
#
namespace = {'session': self}
from twisted.conch.manhole_ssh import (ConchFactory, TerminalRealm,
TerminalSession)
from twisted.conch.manhole import ColoredManhole
class PatchedTerminalSession(TerminalSession):
# get rid of
# exceptions.AttributeError: TerminalSession instance has no attribute 'windowChanged'
def windowChanged(self, winSize):
pass
rlm = TerminalRealm()
rlm.sessionFactory = PatchedTerminalSession # monkey patch
rlm.chainedProtocolFactory.protocolFactory = lambda _: ColoredManhole(
namespace)
ptl = portal.Portal(rlm, [checker])
factory = ConchFactory(ptl)
factory.noisy = False
self._manhole_service = ManholeService(config, details.caller)
starting_topic = '{}.on_manhole_starting'.format(self._uri_prefix)
starting_info = self._manhole_service.marshal()
# the caller gets a progressive result ..
if details.progress:
details.progress(starting_info)
# .. while all others get an event
self.publish(starting_topic,
starting_info,
options=PublishOptions(exclude=details.caller))
try:
self._manhole_service.port = yield create_listening_port_from_config(
config['endpoint'], self.cbdir, factory, self._reactor,
self.log)
except Exception as e:
self._manhole_service = None
emsg = "Manhole service endpoint cannot listen: {}".format(e)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.cannot_listen", emsg)
# alright, manhole has started
self._manhole_service.started = datetime.utcnow()
self._manhole_service.status = 'started'
started_topic = '{}.on_manhole_started'.format(self._uri_prefix)
started_info = self._manhole_service.marshal()
self.publish(started_topic,
started_info,
options=PublishOptions(exclude=details.caller))
returnValue(started_info)
@wamp.register(None)
@inlineCallbacks
def stop_manhole(self, details=None):
"""
Stop the Manhole service running in this process.
This procedure is registered under
* ``crossbar.node.<node_id>.worker.<worker_id>.stop_manhole`` for native workers and under
* ``crossbar.node.<node_id>.controller.stop_manhole`` for node controllers
When no Manhole service is currently running within this process,
or the Manhole service is already shutting down, a
``crossbar.error.not_started`` WAMP error is raised.
The procedure will publish an event when the service **is stopping** to
* ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_stopping`` for native workers and
* ``crossbar.node.<node_id>.controller.on_manhole_stopping`` for node controllers
and will publish an event when the service **has stopped** to
* ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_stopped`` for native workers and
* ``crossbar.node.<node_id>.controller.on_manhole_stopped`` for node controllers
"""
self.log.debug("{cls}.stop_manhole", cls=self.__class__.__name__)
if not self._manhole_service or self._manhole_service.status != 'started':
emsg = "Cannot stop manhole: not running (or already shutting down)"
raise ApplicationError(u"crossbar.error.not_started", emsg)
self._manhole_service.status = 'stopping'
stopping_topic = u'{}.on_manhole_stopping'.format(self._uri_prefix)
stopping_info = None
# the caller gets a progressive result ..
if details.progress:
details.progress(stopping_info)
# .. while all others get an event
self.publish(stopping_topic,
stopping_info,
options=PublishOptions(exclude=details.caller))
try:
yield self._manhole_service.port.stopListening()
except Exception as e:
self.log.warn("error while stop listening on endpoint: {error}",
error=e)
self._manhole_service = None
stopped_topic = u'{}.on_manhole_stopped'.format(self._uri_prefix)
stopped_info = None
self.publish(stopped_topic,
stopped_info,
options=PublishOptions(exclude=details.caller))
returnValue(stopped_info)
@wamp.register(None)
def get_manhole(self, details=None):
"""
Get current manhole service information.
:returns: dict -- A dict with service information or `None` if the service is not running.
"""
self.log.debug("{cls}.get_manhole", cls=self.__class__.__name__)
if not _HAS_MANHOLE:
emsg = "Could not start manhole: required packages are missing ({})".format(
_MANHOLE_MISSING_REASON)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
if not self._manhole_service:
return None
else:
return self._manhole_service.marshal()
@wamp.register(None)
def utcnow(self, details=None):
"""
Return current time as determined from within this process.
**Usage:**
This procedure is registered under
* ``crossbar.node.<node_id>.worker.<worker_id>.utcnow`` for native workers and under
* ``crossbar.node.<node_id>.controller.utcnow`` for node controllers
:returns: Current time (UTC) in UTC ISO 8601 format.
:rtype: unicode
"""
self.log.debug("{cls}.utcnow", cls=self.__class__.__name__)
return utcnow()
@wamp.register(None)
def started(self, details=None):
"""
Return start time of this process.
**Usage:**
This procedure is registered under
* ``crossbar.node.<node_id>.worker.<worker_id>.started`` for native workers and under
* ``crossbar.node.<node_id>.controller.started`` for node controllers
:returns: Start time (UTC) in UTC ISO 8601 format.
:rtype: unicode
"""
self.log.debug("{cls}.started", cls=self.__class__.__name__)
return utcstr(self._started)
@wamp.register(None)
def uptime(self, details=None):
"""
Return uptime of this process.
**Usage:**
This procedure is registered under
* ``crossbar.node.<node_id>.worker.<worker_id>.uptime`` for native workers and under
* ``crossbar.node.<node_id>.controller.uptime`` for node controllers
:returns: Uptime in seconds.
:rtype: float
"""
self.log.debug("{cls}.uptime", cls=self.__class__.__name__)
now = datetime.utcnow()
return (now - self._started).total_seconds()
class NativeProcessSession(ApplicationSession):
"""
A native Crossbar.io process (currently: controller, router or container).
"""
log = make_logger()
def __init__(self, config=None, reactor=None):
if not reactor:
from twisted.internet import reactor
self._reactor = reactor
self._reactor = reactor
super(ApplicationSession, self).__init__(config=config)
def onConnect(self, do_join=True):
"""
"""
if not hasattr(self, 'cbdir'):
self.cbdir = self.config.extra.cbdir
if not hasattr(self, '_uri_prefix'):
self._uri_prefix = 'crossbar.node.{}'.format(
self.config.extra.node)
self._started = datetime.utcnow()
# see: BaseSession
self.include_traceback = False
self.debug_app = False
self._manhole_service = None
if _HAS_PSUTIL:
self._pinfo = ProcessInfo()
self._pinfo_monitor = None
self._pinfo_monitor_seq = 0
else:
self._pinfo = None
self._pinfo_monitor = None
self._pinfo_monitor_seq = None
self.log.info("Process utilities not available")
self._connections = {}
if do_join:
self.join(self.config.realm)
@inlineCallbacks
def onJoin(self, details):
"""
Called when process has joined the node's management realm.
"""
procs = [
'start_manhole', 'stop_manhole', 'get_manhole', 'start_connection',
'stop_connection', 'get_connections', 'trigger_gc', 'utcnow',
'started', 'uptime', 'get_process_info', 'get_process_stats',
'set_process_stats_monitoring'
]
dl = []
for proc in procs:
uri = '{}.{}'.format(self._uri_prefix, proc)
self.log.debug("Registering procedure '{uri}'", uri=uri)
dl.append(
self.register(getattr(self, proc),
uri,
options=RegisterOptions(details_arg='details')))
regs = yield DeferredList(dl)
self.log.debug("Registered {len_reg} procedures", len_reg=len(regs))
@inlineCallbacks
def start_connection(self, id, config, details=None):
"""
Starts a connection in this process.
:param id: The ID for the started connection.
:type id: unicode
:param config: Connection configuration.
:type config: dict
:param details: Caller details.
:type details: instance of :class:`autobahn.wamp.types.CallDetails`
:returns dict -- The connection.
"""
self.log.debug("start_connection: id={id}, config={config}",
id=id,
config=config)
# prohibit starting a component twice
#
if id in self._connections:
emsg = "cannot start connection: a connection with id={} is already started".format(
id)
self.log.warn(emsg)
raise ApplicationError("crossbar.error.invalid_configuration",
emsg)
# check configuration
#
try:
checkconfig.check_connection(config)
except Exception as e:
emsg = "invalid connection configuration ({})".format(e)
self.log.warn(emsg)
raise ApplicationError("crossbar.error.invalid_configuration",
emsg)
else:
self.log.info("Starting {} in process.".format(config['type']))
if config['type'] == u'postgresql.connection':
if _HAS_POSTGRESQL:
connection = PostgreSQLConnection(id, config)
else:
emsg = "unable to start connection - required PostgreSQL driver package not installed"
self.log.warn(emsg)
raise ApplicationError("crossbar.error.feature_unavailable",
emsg)
else:
# should not arrive here
raise Exception("logic error")
self._connections[id] = connection
try:
yield connection.start()
self.log.info(
"Connection {connection_type} started '{connection_id}'",
connection_id=id,
connection_type=config['type'])
except Exception as e:
del self._connections[id]
raise
state = connection.marshal()
self.publish(u'crossbar.node.process.on_connection_start', state)
returnValue(state)
@inlineCallbacks
def stop_connection(self, id, details=None):
"""
Stop a connection currently running within this process.
:param id: The ID of the connection to stop.
:type id: unicode
:param details: Caller details.
:type details: instance of :class:`autobahn.wamp.types.CallDetails`
:returns dict -- A dict with component start information.
"""
self.log.debug("stop_connection: id={id}", id=id)
if id not in self._connections:
raise ApplicationError(
'crossbar.error.no_such_object',
'no connection with ID {} running in this process'.format(id))
connection = self._connections[id]
try:
yield connection.stop()
except Exception as e:
self.log.warn('could not stop connection {id}: {error}', error=e)
raise
del self._connections[id]
state = connection.marshal()
self.publish(u'crossbar.node.process.on_connection_stop', state)
returnValue(state)
def get_connections(self, details=None):
"""
Get connections currently running within this processs.
:param details: Caller details.
:type details: instance of :class:`autobahn.wamp.types.CallDetails`
:returns list -- List of connections.
"""
self.log.debug("get_connections")
res = []
for c in self._connections.values():
res.append(c.marshal())
return res
def get_process_info(self, details=None):
"""
Get process information (open files, sockets, ...).
:returns: dict -- Dictionary with process information.
"""
self.log.debug("{cls}.get_process_info", cls=self.__class__.__name__)
if self._pinfo:
return self._pinfo.get_info()
else:
emsg = "ERROR: could not retrieve process statistics - required packages not installed"
raise ApplicationError("crossbar.error.feature_unavailable", emsg)
def get_process_stats(self, details=None):
"""
Get process statistics (CPU, memory, I/O).
:returns: dict -- Dictionary with process statistics.
"""
self.log.debug("{cls}.get_process_stats", cls=self.__class__.__name__)
if self._pinfo:
return self._pinfo.get_stats()
else:
emsg = "ERROR: could not retrieve process statistics - required packages not installed"
raise ApplicationError("crossbar.error.feature_unavailable", emsg)
def set_process_stats_monitoring(self, interval, details=None):
"""
Enable/disable periodic publication of process statistics.
:param interval: The monitoring interval in seconds. Set to 0 to disable monitoring.
:type interval: float
"""
self.log.debug(
"{cls}.set_process_stats_monitoring(interval = {interval})",
cls=self.__class__.__name__,
interval=interval)
if self._pinfo:
stats_monitor_set_topic = '{}.on_process_stats_monitoring_set'.format(
self._uri_prefix)
# stop and remove any existing monitor
if self._pinfo_monitor:
self._pinfo_monitor.stop()
self._pinfo_monitor = None
self.publish(stats_monitor_set_topic,
0,
options=PublishOptions(exclude=[details.caller]))
# possibly start a new monitor
if interval > 0:
stats_topic = '{}.on_process_stats'.format(self._uri_prefix)
def publish_stats():
stats = self._pinfo.get_stats()
self._pinfo_monitor_seq += 1
stats['seq'] = self._pinfo_monitor_seq
self.publish(stats_topic, stats)
self._pinfo_monitor = LoopingCall(publish_stats)
self._pinfo_monitor.start(interval)
self.publish(stats_monitor_set_topic,
interval,
options=PublishOptions(exclude=[details.caller]))
else:
emsg = "ERROR: cannot setup process statistics monitor - required packages not installed"
raise ApplicationError("crossbar.error.feature_unavailable", emsg)
def trigger_gc(self, details=None):
"""
Triggers a garbage collection.
:returns: float -- Time consumed for GC in ms.
"""
self.msg.debug("{cls}.trigger_gc", cls=self.__class__.__name__)
started = rtime()
gc.collect()
return 1000. * (rtime() - started)
@inlineCallbacks
def start_manhole(self, config, details=None):
"""
Start a manhole (SSH) within this worker.
:param config: Manhole configuration.
:type config: obj
"""
self.log.debug("{cls}.start_manhole(config = {config})",
cls=self.__class__.__name__,
config=config)
if not _HAS_MANHOLE:
emsg = "ERROR: could not start manhole - required packages are missing ({})".format(
_MANHOLE_MISSING_REASON)
self.log.error(emsg)
raise ApplicationError("crossbar.error.feature_unavailable", emsg)
if self._manhole_service:
emsg = "ERROR: could not start manhole - already running (or starting)"
self.log.warn(emsg)
raise ApplicationError("crossbar.error.already_started", emsg)
try:
checkconfig.check_manhole(config)
except Exception as e:
emsg = "ERROR: could not start manhole - invalid configuration ({})".format(
e)
self.log.error(emsg)
raise ApplicationError('crossbar.error.invalid_configuration',
emsg)
# setup user authentication
#
checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()
for user in config['users']:
checker.addUser(user['user'], user['password'])
# setup manhole namespace
#
namespace = {'session': self}
class PatchedTerminalSession(TerminalSession):
# get rid of
# exceptions.AttributeError: TerminalSession instance has no attribute 'windowChanged'
def windowChanged(self, winSize):
pass
rlm = TerminalRealm()
rlm.sessionFactory = PatchedTerminalSession # monkey patch
rlm.chainedProtocolFactory.protocolFactory = lambda _: ColoredManhole(
namespace)
ptl = portal.Portal(rlm, [checker])
factory = ConchFactory(ptl)
factory.noisy = False
self._manhole_service = ManholeService(config, details.caller)
starting_topic = '{}.on_manhole_starting'.format(self._uri_prefix)
starting_info = self._manhole_service.marshal()
# the caller gets a progressive result ..
if details.progress:
details.progress(starting_info)
# .. while all others get an event
self.publish(starting_topic,
starting_info,
options=PublishOptions(exclude=[details.caller]))
try:
self._manhole_service.port = yield create_listening_port_from_config(
config['endpoint'], factory, self.cbdir, self._reactor)
except Exception as e:
self._manhole_service = None
emsg = "ERROR: manhole service endpoint cannot listen - {}".format(
e)
self.log.error(emsg)
raise ApplicationError("crossbar.error.cannot_listen", emsg)
# alright, manhole has started
self._manhole_service.started = datetime.utcnow()
self._manhole_service.status = 'started'
started_topic = '{}.on_manhole_started'.format(self._uri_prefix)
started_info = self._manhole_service.marshal()
self.publish(started_topic,
started_info,
options=PublishOptions(exclude=[details.caller]))
returnValue(started_info)
@inlineCallbacks
def stop_manhole(self, details=None):
"""
Stop Manhole.
"""
self.log.debug("{cls}.stop_manhole", cls=self.__class__.__name__)
if not _HAS_MANHOLE:
emsg = "ERROR: could not start manhole - required packages are missing ({})".format(
_MANHOLE_MISSING_REASON)
self.log.error(emsg)
raise ApplicationError("crossbar.error.feature_unavailable", emsg)
if not self._manhole_service or self._manhole_service.status != 'started':
emsg = "ERROR: cannot stop manhole - not running (or already shutting down)"
raise ApplicationError("crossbar.error.not_started", emsg)
self._manhole_service.status = 'stopping'
stopping_topic = '{}.on_manhole_stopping'.format(self._uri_prefix)
stopping_info = None
# the caller gets a progressive result ..
if details.progress:
details.progress(stopping_info)
# .. while all others get an event
self.publish(stopping_topic,
stopping_info,
options=PublishOptions(exclude=[details.caller]))
try:
yield self._manhole_service.port.stopListening()
except Exception as e:
raise Exception(
"INTERNAL ERROR: don't know how to handle a failed called to stopListening() - {}"
.format(e))
self._manhole_service = None
stopped_topic = '{}.on_manhole_stopped'.format(self._uri_prefix)
stopped_info = None
self.publish(stopped_topic,
stopped_info,
options=PublishOptions(exclude=[details.caller]))
returnValue(stopped_info)
def get_manhole(self, details=None):
"""
Get current manhole service information.
:returns: dict -- A dict with service information or `None` if the service is not running.
"""
self.log.debug("{cls}.get_manhole", cls=self.__class__.__name__)
if not _HAS_MANHOLE:
emsg = "ERROR: could not start manhole - required packages are missing ({})".format(
_MANHOLE_MISSING_REASON)
self.log.error(emsg)
raise ApplicationError("crossbar.error.feature_unavailable", emsg)
if not self._manhole_service:
return None
else:
return self._manhole_service.marshal()
def utcnow(self, details=None):
"""
Return current time as determined from within this process.
:returns str -- Current time (UTC) in UTC ISO 8601 format.
"""
self.log.debug("{cls}.utcnow", cls=self.__class__.__name__)
return utcnow()
def started(self, details=None):
"""
Return start time of this process.
:returns str -- Start time (UTC) in UTC ISO 8601 format.
"""
self.log.debug("{cls}.started", cls=self.__class__.__name__)
return utcstr(self._started)
def uptime(self, details=None):
"""
Uptime of this process.
:returns float -- Uptime in seconds.
"""
self.log.debug("{cls}.uptime", cls=self.__class__.__name__)
now = datetime.utcnow()
return (now - self._started).total_seconds()
class NativeProcessSession(ApplicationSession):
"""
A native Crossbar.io process (currently: controller, router or container).
"""
log = make_logger()
def __init__(self, config=None, reactor=None):
if not reactor:
from twisted.internet import reactor
self._reactor = reactor
self._reactor = reactor
super(ApplicationSession, self).__init__(config=config)
def onConnect(self, do_join=True):
"""
"""
if not hasattr(self, 'cbdir'):
self.cbdir = self.config.extra.cbdir
if not hasattr(self, '_uri_prefix'):
self._uri_prefix = 'crossbar.node.{}'.format(self.config.extra.node)
self._started = datetime.utcnow()
# see: BaseSession
self.include_traceback = False
self.debug_app = False
self._manhole_service = None
if _HAS_PSUTIL:
self._pinfo = ProcessInfo()
self._pinfo_monitor = None
self._pinfo_monitor_seq = 0
else:
self._pinfo = None
self._pinfo_monitor = None
self._pinfo_monitor_seq = None
self.log.info("Process utilities not available")
self._connections = {}
if do_join:
self.join(self.config.realm)
@inlineCallbacks
def onJoin(self, details):
"""
Called when process has joined the node's management realm.
"""
procs = [
'start_manhole',
'stop_manhole',
'get_manhole',
'start_connection',
'stop_connection',
'get_connections',
'trigger_gc',
'utcnow',
'started',
'uptime',
'get_process_info',
'get_process_stats',
'set_process_stats_monitoring'
]
dl = []
for proc in procs:
uri = '{}.{}'.format(self._uri_prefix, proc)
self.log.debug("Registering procedure '{uri}'", uri=uri)
dl.append(self.register(getattr(self, proc), uri, options=RegisterOptions(details_arg='details')))
regs = yield DeferredList(dl)
self.log.debug("Registered {len_reg} procedures", len_reg=len(regs))
@inlineCallbacks
def start_connection(self, id, config, details=None):
"""
Starts a connection in this process.
:param id: The ID for the started connection.
:type id: unicode
:param config: Connection configuration.
:type config: dict
:param details: Caller details.
:type details: instance of :class:`autobahn.wamp.types.CallDetails`
:returns dict -- The connection.
"""
self.log.debug("start_connection: id={id}, config={config}", id=id, config=config)
# prohibit starting a component twice
#
if id in self._connections:
emsg = "cannot start connection: a connection with id={} is already started".format(id)
self.log.warn(emsg)
raise ApplicationError(u"crossbar.error.invalid_configuration", emsg)
# check configuration
#
try:
checkconfig.check_connection(config)
except Exception as e:
emsg = "invalid connection configuration ({})".format(e)
self.log.warn(emsg)
raise ApplicationError(u"crossbar.error.invalid_configuration", emsg)
else:
self.log.info("Starting {} in process.".format(config['type']))
if config['type'] == u'postgresql.connection':
if _HAS_POSTGRESQL:
connection = PostgreSQLConnection(id, config)
else:
emsg = "unable to start connection - required PostgreSQL driver package not installed"
self.log.warn(emsg)
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
else:
# should not arrive here
raise Exception("logic error")
self._connections[id] = connection
try:
yield connection.start()
self.log.info("Connection {connection_type} started '{connection_id}'", connection_id=id, connection_type=config['type'])
except Exception as e:
del self._connections[id]
raise
state = connection.marshal()
self.publish(u'crossbar.node.process.on_connection_start', state)
returnValue(state)
@inlineCallbacks
def stop_connection(self, id, details=None):
"""
Stop a connection currently running within this process.
:param id: The ID of the connection to stop.
:type id: unicode
:param details: Caller details.
:type details: instance of :class:`autobahn.wamp.types.CallDetails`
:returns dict -- A dict with component start information.
"""
self.log.debug("stop_connection: id={id}", id=id)
if id not in self._connections:
raise ApplicationError(u'crossbar.error.no_such_object', 'no connection with ID {} running in this process'.format(id))
connection = self._connections[id]
try:
yield connection.stop()
except Exception as e:
self.log.warn('could not stop connection {id}: {error}', error=e)
raise
del self._connections[id]
state = connection.marshal()
self.publish(u'crossbar.node.process.on_connection_stop', state)
returnValue(state)
def get_connections(self, details=None):
"""
Get connections currently running within this processs.
:param details: Caller details.
:type details: instance of :class:`autobahn.wamp.types.CallDetails`
:returns list -- List of connections.
"""
self.log.debug("get_connections")
res = []
for c in self._connections.values():
res.append(c.marshal())
return res
def get_process_info(self, details=None):
"""
Get process information (open files, sockets, ...).
:returns: dict -- Dictionary with process information.
"""
self.log.debug("{cls}.get_process_info",
cls=self.__class__.__name__)
if self._pinfo:
return self._pinfo.get_info()
else:
emsg = "Could not retrieve process statistics: required packages not installed"
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
def get_process_stats(self, details=None):
"""
Get process statistics (CPU, memory, I/O).
:returns: dict -- Dictionary with process statistics.
"""
self.log.debug("{cls}.get_process_stats", cls=self.__class__.__name__)
if self._pinfo:
return self._pinfo.get_stats()
else:
emsg = "Could not retrieve process statistics: required packages not installed"
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
def set_process_stats_monitoring(self, interval, details=None):
"""
Enable/disable periodic publication of process statistics.
:param interval: The monitoring interval in seconds. Set to 0 to disable monitoring.
:type interval: float
"""
self.log.debug("{cls}.set_process_stats_monitoring(interval = {interval})",
cls=self.__class__.__name__, interval=interval)
if self._pinfo:
stats_monitor_set_topic = '{}.on_process_stats_monitoring_set'.format(self._uri_prefix)
# stop and remove any existing monitor
if self._pinfo_monitor:
self._pinfo_monitor.stop()
self._pinfo_monitor = None
self.publish(stats_monitor_set_topic, 0, options=PublishOptions(exclude=details.caller))
# possibly start a new monitor
if interval > 0:
stats_topic = '{}.on_process_stats'.format(self._uri_prefix)
def publish_stats():
stats = self._pinfo.get_stats()
self._pinfo_monitor_seq += 1
stats[u'seq'] = self._pinfo_monitor_seq
self.publish(stats_topic, stats)
self._pinfo_monitor = LoopingCall(publish_stats)
self._pinfo_monitor.start(interval)
self.publish(stats_monitor_set_topic, interval, options=PublishOptions(exclude=details.caller))
else:
emsg = "Cannot setup process statistics monitor: required packages not installed"
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
def trigger_gc(self, details=None):
"""
Manually trigger a garbage collection in this native process.
This procedure is registered under
``crossbar.node.<node_id>.worker.<worker_id>.trigger_gc``
for native workers and under
``crossbar.node.<node_id>.controller.trigger_gc``
for node controllers.
The procedure will publish an event when the garabage collection has finished to
``crossbar.node.<node_id>.worker.<worker_id>.on_gc_finished``
for native workers and
``crossbar.node.<node_id>.controller.on_gc_finished``
for node controllers:
.. code-block:: javascript
{
"requester": {
"session_id": 982734923,
"auth_id": "bob",
"auth_role": "admin"
},
"duration": 190
}
.. note:: The caller of this procedure will NOT receive the event.
:returns: Time (wall clock) consumed for garbage collection in ms.
:rtype: int
"""
self.log.debug("{cls}.trigger_gc", cls=self.__class__.__name__)
started = rtime()
# now trigger GC .. this is blocking!
gc.collect()
duration = int(round(1000. * (rtime() - started)))
on_gc_finished = u'{}.on_gc_finished'.format(self._uri_prefix)
self.publish(
on_gc_finished,
{
u'requester': {
u'session_id': details.caller,
# FIXME:
u'auth_id': None,
u'auth_role': None
},
u'duration': duration
},
options=PublishOptions(exclude=details.caller)
)
return duration
@inlineCallbacks
def start_manhole(self, config, details=None):
"""
Start a Manhole service within this process.
**Usage:**
This procedure is registered under
* ``crossbar.node.<node_id>.worker.<worker_id>.start_manhole`` - for native workers
* ``crossbar.node.<node_id>.controller.start_manhole`` - for node controllers
The procedure takes a Manhole service configuration which defines
a listening endpoint for the service and a list of users including
passwords, e.g.
.. code-block:: javascript
{
"endpoint": {
"type": "tcp",
"port": 6022
},
"users": [
{
"user": "******",
"password": "******"
}
]
}
**Errors:**
The procedure may raise the following errors:
* ``crossbar.error.invalid_configuration`` - the provided configuration is invalid
* ``crossbar.error.already_started`` - the Manhole service is already running (or starting)
* ``crossbar.error.feature_unavailable`` - the required support packages are not installed
**Events:**
The procedure will publish an event when the service **is starting** to
* ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_starting`` - for native workers
* ``crossbar.node.<node_id>.controller.on_manhole_starting`` - for node controllers
and publish an event when the service **has started** to
* ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_started`` - for native workers
* ``crossbar.node.<node_id>.controller.on_manhole_started`` - for node controllers
:param config: Manhole service configuration.
:type config: dict
"""
self.log.debug("{cls}.start_manhole(config = {config})",
cls=self.__class__.__name__, config=config)
if not _HAS_MANHOLE:
emsg = "Could not start manhole: required packages are missing ({})".format(_MANHOLE_MISSING_REASON)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
if self._manhole_service:
emsg = "Could not start manhole - already running (or starting)"
self.log.warn(emsg)
raise ApplicationError(u"crossbar.error.already_started", emsg)
try:
checkconfig.check_manhole(config)
except Exception as e:
emsg = "Could not start manhole: invalid configuration ({})".format(e)
self.log.error(emsg)
raise ApplicationError(u'crossbar.error.invalid_configuration', emsg)
# setup user authentication
#
checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()
for user in config['users']:
checker.addUser(user['user'], user['password'])
# setup manhole namespace
#
namespace = {'session': self}
class PatchedTerminalSession(TerminalSession):
# get rid of
# exceptions.AttributeError: TerminalSession instance has no attribute 'windowChanged'
def windowChanged(self, winSize):
pass
rlm = TerminalRealm()
rlm.sessionFactory = PatchedTerminalSession # monkey patch
rlm.chainedProtocolFactory.protocolFactory = lambda _: ColoredManhole(namespace)
ptl = portal.Portal(rlm, [checker])
factory = ConchFactory(ptl)
factory.noisy = False
self._manhole_service = ManholeService(config, details.caller)
starting_topic = '{}.on_manhole_starting'.format(self._uri_prefix)
starting_info = self._manhole_service.marshal()
# the caller gets a progressive result ..
if details.progress:
details.progress(starting_info)
# .. while all others get an event
self.publish(starting_topic, starting_info, options=PublishOptions(exclude=details.caller))
try:
self._manhole_service.port = yield create_listening_port_from_config(config['endpoint'],
self.cbdir,
factory,
self._reactor,
self.log)
except Exception as e:
self._manhole_service = None
emsg = "Manhole service endpoint cannot listen: {}".format(e)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.cannot_listen", emsg)
# alright, manhole has started
self._manhole_service.started = datetime.utcnow()
self._manhole_service.status = 'started'
started_topic = '{}.on_manhole_started'.format(self._uri_prefix)
started_info = self._manhole_service.marshal()
self.publish(started_topic, started_info, options=PublishOptions(exclude=details.caller))
returnValue(started_info)
@inlineCallbacks
def stop_manhole(self, details=None):
"""
Stop the Manhole service running in this process.
This procedure is registered under
* ``crossbar.node.<node_id>.worker.<worker_id>.stop_manhole`` for native workers and under
* ``crossbar.node.<node_id>.controller.stop_manhole`` for node controllers
When no Manhole service is currently running within this process,
or the Manhole service is already shutting down, a
``crossbar.error.not_started`` WAMP error is raised.
The procedure will publish an event when the service **is stopping** to
* ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_stopping`` for native workers and
* ``crossbar.node.<node_id>.controller.on_manhole_stopping`` for node controllers
and will publish an event when the service **has stopped** to
* ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_stopped`` for native workers and
* ``crossbar.node.<node_id>.controller.on_manhole_stopped`` for node controllers
"""
self.log.debug("{cls}.stop_manhole", cls=self.__class__.__name__)
if not self._manhole_service or self._manhole_service.status != 'started':
emsg = "Cannot stop manhole: not running (or already shutting down)"
raise ApplicationError(u"crossbar.error.not_started", emsg)
self._manhole_service.status = 'stopping'
stopping_topic = u'{}.on_manhole_stopping'.format(self._uri_prefix)
stopping_info = None
# the caller gets a progressive result ..
if details.progress:
details.progress(stopping_info)
# .. while all others get an event
self.publish(stopping_topic, stopping_info, options=PublishOptions(exclude=details.caller))
try:
yield self._manhole_service.port.stopListening()
except Exception as e:
self.log.warn("error while stop listening on endpoint: {error}", error=e)
self._manhole_service = None
stopped_topic = u'{}.on_manhole_stopped'.format(self._uri_prefix)
stopped_info = None
self.publish(stopped_topic, stopped_info, options=PublishOptions(exclude=details.caller))
returnValue(stopped_info)
def get_manhole(self, details=None):
"""
Get current manhole service information.
:returns: dict -- A dict with service information or `None` if the service is not running.
"""
self.log.debug("{cls}.get_manhole", cls=self.__class__.__name__)
if not _HAS_MANHOLE:
emsg = "Could not start manhole: required packages are missing ({})".format(_MANHOLE_MISSING_REASON)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
if not self._manhole_service:
return None
else:
return self._manhole_service.marshal()
def utcnow(self, details=None):
"""
Return current time as determined from within this process.
**Usage:**
This procedure is registered under
* ``crossbar.node.<node_id>.worker.<worker_id>.utcnow`` for native workers and under
* ``crossbar.node.<node_id>.controller.utcnow`` for node controllers
:returns: Current time (UTC) in UTC ISO 8601 format.
:rtype: unicode
"""
self.log.debug("{cls}.utcnow", cls=self.__class__.__name__)
return utcnow()
def started(self, details=None):
"""
Return start time of this process.
**Usage:**
This procedure is registered under
* ``crossbar.node.<node_id>.worker.<worker_id>.started`` for native workers and under
* ``crossbar.node.<node_id>.controller.started`` for node controllers
:returns: Start time (UTC) in UTC ISO 8601 format.
:rtype: unicode
"""
self.log.debug("{cls}.started", cls=self.__class__.__name__)
return utcstr(self._started)
def uptime(self, details=None):
"""
Return uptime of this process.
**Usage:**
This procedure is registered under
* ``crossbar.node.<node_id>.worker.<worker_id>.uptime`` for native workers and under
* ``crossbar.node.<node_id>.controller.uptime`` for node controllers
:returns: Uptime in seconds.
:rtype: float
"""
self.log.debug("{cls}.uptime", cls=self.__class__.__name__)
now = datetime.utcnow()
return (now - self._started).total_seconds()
class AppSession(ApplicationSession):
@inlineCallbacks
def onJoin(self, details):
self._db = Database(dbpath='../results')
self._schema = Schema.attach(self._db)
self._pinfo = ProcessInfo()
self._logname = self.config.extra['logname']
self._period = self.config.extra.get('period', 5.)
self._running = True
dl = []
for i in range(8):
d = self._loop(i)
dl.append(d)
d = gatherResults(dl)
try:
yield d
except TransportLost:
pass
def onLeave(self, details):
self._running = False
@inlineCallbacks
def _loop(self, index):
prefix = self.config.extra['prefix']
last = None
while self._running:
rtts = []
batch_started_str = utcnow()
batch_started = time()
while (time() - batch_started) < self._period:
ts_req = time_ns()
res = yield self.call('{}.echo'.format(prefix), ts_req)
ts_res = time_ns()
assert res == ts_req
rtt = ts_res - ts_req
rtts.append(rtt)
stats = self._pinfo.get_stats()
if last:
batch_duration = (stats['time'] - last['time']) / 10**9
ctx = round(
(stats['voluntary'] - last['voluntary']) / batch_duration,
0)
self.log.info('{logname}: {cpu} cpu, {mem} mem, {ctx} ctx',
logname=self._logname,
cpu=round(stats['cpu_percent'], 1),
mem=round(stats['mem_percent'], 1),
ctx=ctx)
rtts = sorted(rtts)
sr = WampStatsRecord()
sr.key = '{}#{}.{}'.format(batch_started_str, self._logname,
index)
sr.count = len(rtts)
sr.calls_per_sec = int(round(sr.count / batch_duration, 0))
# all times here are in microseconds:
sr.avg_rtt = round(1000000. * batch_duration / float(sr.count),
1)
sr.max_rtt = round(rtts[-1] / 1000, 1)
sr.q50_rtt = round(rtts[int(sr.count / 2.)] / 1000, 1)
sr.q99_rtt = round(rtts[int(-(sr.count / 100.))] / 1000, 1)
sr.q995_rtt = round(rtts[int(-(sr.count / 995.))] / 1000, 1)
with self._db.begin(write=True) as txn:
self._schema.wamp_stats[txn, sr.key] = sr
print(
"{}: {} calls, {} calls/sec, RTT (us): q50 {}, avg {}, q99 {}, q995 {}, max {}"
.format(sr.key, sr.count, sr.calls_per_sec, sr.q50_rtt,
sr.avg_rtt, sr.q99_rtt, sr.q995_rtt, sr.max_rtt))
last = stats
class NativeProcess(ApplicationSession):
"""
A native Crossbar.io process (currently: controller, router or container).
"""
log = make_logger()
WORKER_TYPE = u'native'
def onUserError(self, fail, msg):
"""
Implements :func:`autobahn.wamp.interfaces.ISession.onUserError`
"""
if isinstance(fail.value, ApplicationError):
self.log.debug('{klass}.onUserError(): "{msg}"',
klass=self.__class__.__name__,
msg=fail.value.error_message())
else:
self.log.error(
'{klass}.onUserError(): "{msg}"\n{traceback}',
klass=self.__class__.__name__,
msg=msg,
traceback=txaio.failure_format_traceback(fail),
)
def __init__(self, config=None, reactor=None, personality=None):
# Twisted reactor
if not reactor:
from twisted.internet import reactor
self._reactor = reactor
self._reactor = reactor
# node/software personality
if personality:
self.personality = personality
else:
from crossbar.personality import Personality
self.personality = Personality
self._node_id = config.extra.node if config and config.extra else None
self._worker_id = config.extra.worker if config and config.extra else None
self._uri_prefix = u'crossbar.worker.{}'.format(self._worker_id)
# base ctor
super(ApplicationSession, self).__init__(config=config)
def onConnect(self, do_join=True):
if not hasattr(self, 'cbdir'):
self.cbdir = self.config.extra.cbdir
if not hasattr(self, '_uri_prefix'):
self._uri_prefix = 'crossbar.node.{}'.format(self.config.extra.node)
self._started = datetime.utcnow()
# see: BaseSession
self.include_traceback = False
self._manhole_service = None
if _HAS_PSUTIL:
self._pinfo = ProcessInfo()
self._pmonitor = ProcessMonitor(self.WORKER_TYPE, {})
self._pinfo_monitor = None
self._pinfo_monitor_seq = 0
else:
self._pinfo = None
self._pmonitor = None
self._pinfo_monitor = None
self._pinfo_monitor_seq = None
self.log.info("Process utilities not available")
if do_join:
self.join(self.config.realm)
@inlineCallbacks
def onJoin(self, details):
"""
Called when process has joined the node's management realm.
"""
regs = yield self.register(
self,
prefix=u'{}.'.format(self._uri_prefix),
options=RegisterOptions(details_arg='details'),
)
self.log.info("Registered {len_reg} procedures", len_reg=len(regs))
for reg in regs:
if isinstance(reg, Failure):
self.log.error("Failed to register: {f}", f=reg, log_failure=reg)
else:
self.log.debug(' {proc}', proc=reg.procedure)
returnValue(regs)
@wamp.register(None)
def get_cpu_count(self, logical=True, details=None):
"""
Returns the CPU core count on the machine this process is running on.
:param logical: If enabled (default), include logical CPU cores ("Hyperthreading"),
else only count physical CPU cores.
:type logical: bool
:returns: The number of CPU cores.
:rtype: int
"""
if not _HAS_PSUTIL:
emsg = "unable to get CPU count: required package 'psutil' is not installed"
self.log.warn(emsg)
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
return psutil.cpu_count(logical=logical)
@wamp.register(None)
def get_cpus(self, details=None):
"""
:returns: List of CPU IDs.
:rtype: list[int]
"""
if not _HAS_PSUTIL:
emsg = "unable to get CPUs: required package 'psutil' is not installed"
self.log.warn(emsg)
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
return self._pinfo.cpus
@wamp.register(None)
def get_cpu_affinity(self, details=None):
"""
Get CPU affinity of this process.
:returns: List of CPU IDs the process affinity is set to.
:rtype: list[int]
"""
if not _HAS_PSUTIL:
emsg = "unable to get CPU affinity: required package 'psutil' is not installed"
self.log.warn(emsg)
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
try:
p = psutil.Process(os.getpid())
current_affinity = p.cpu_affinity()
except Exception as e:
emsg = "Could not get CPU affinity: {}".format(e)
self.log.failure(emsg)
raise ApplicationError(u"crossbar.error.runtime_error", emsg)
else:
return current_affinity
@wamp.register(None)
def set_cpu_affinity(self, cpus, relative=True, details=None):
"""
Set CPU affinity of this process.
:param cpus: List of CPU IDs to set process affinity to. Each CPU ID must be
from the list `[0 .. N_CPUs]`, where N_CPUs can be retrieved via
``crossbar.worker.<worker_id>.get_cpu_count``.
:type cpus: list[int]
:returns: List of CPU IDs the process affinity is set to.
:rtype: list[int]
"""
if not _HAS_PSUTIL:
emsg = "Unable to set CPU affinity: required package 'psutil' is not installed"
self.log.warn(emsg)
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
if sys.platform.startswith('darwin'):
# https://superuser.com/questions/149312/how-to-set-processor-affinity-on-os-x
emsg = "Unable to set CPU affinity: OSX lacks process CPU affinity"
self.log.warn(emsg)
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
if relative:
_cpu_ids = self._pinfo.cpus
_cpus = [_cpu_ids[i] for i in cpus]
else:
_cpus = cpus
try:
p = psutil.Process(os.getpid())
p.cpu_affinity(_cpus)
new_affinity = p.cpu_affinity()
if set(_cpus) != set(new_affinity):
raise Exception('CPUs mismatch after affinity setting ({} != {})'.format(set(_cpus), set(new_affinity)))
except Exception as e:
emsg = "Could not set CPU affinity: {}".format(e)
self.log.failure(emsg)
raise ApplicationError(u"crossbar.error.runtime_error", emsg)
else:
# publish info to all but the caller ..
#
cpu_affinity_set_topic = u'{}.on_cpu_affinity_set'.format(self._uri_prefix)
cpu_affinity_set_info = {
u'cpus': cpus,
u'relative': relative,
u'affinity': new_affinity,
u'who': details.caller
}
self.publish(cpu_affinity_set_topic, cpu_affinity_set_info, options=PublishOptions(exclude=details.caller))
# .. and return info directly to caller
#
return new_affinity
@wamp.register(None)
def get_process_info(self, details=None):
"""
Get process information (open files, sockets, ...).
:returns: Dictionary with process information.
"""
self.log.debug("{cls}.get_process_info",
cls=self.__class__.__name__)
if self._pinfo:
# psutil.AccessDenied
# PermissionError: [Errno 13] Permission denied: '/proc/14787/io'
return self._pinfo.get_info()
else:
emsg = "Could not retrieve process statistics: required packages not installed"
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
@wamp.register(None)
def get_process_stats(self, details=None):
"""
Get process statistics (CPU, memory, I/O).
:returns: Dictionary with process statistics.
"""
self.log.debug("{cls}.get_process_stats", cls=self.__class__.__name__)
if self._pinfo:
return self._pinfo.get_stats()
else:
emsg = "Could not retrieve process statistics: required packages not installed"
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
@wamp.register(None)
def get_process_monitor(self, details=None):
self.log.debug("{cls}.get_process_monitor", cls=self.__class__.__name__)
if self._pmonitor:
return self._pmonitor.poll()
else:
emsg = "Could not retrieve process statistics: required packages not installed"
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
@wamp.register(None)
def set_process_stats_monitoring(self, interval, details=None):
"""
Enable/disable periodic publication of process statistics.
:param interval: The monitoring interval in seconds. Set to 0 to disable monitoring.
:type interval: float
"""
self.log.debug("{cls}.set_process_stats_monitoring(interval = {interval})",
cls=self.__class__.__name__, interval=interval)
if self._pinfo:
stats_monitor_set_topic = '{}.on_process_stats_monitoring_set'.format(self._uri_prefix)
# stop and remove any existing monitor
if self._pinfo_monitor:
self._pinfo_monitor.stop()
self._pinfo_monitor = None
self.publish(stats_monitor_set_topic, 0, options=PublishOptions(exclude=details.caller))
# possibly start a new monitor
if interval > 0:
stats_topic = '{}.on_process_stats'.format(self._uri_prefix)
def publish_stats():
stats = self._pinfo.get_stats()
self._pinfo_monitor_seq += 1
stats[u'seq'] = self._pinfo_monitor_seq
self.publish(stats_topic, stats)
self._pinfo_monitor = LoopingCall(publish_stats)
self._pinfo_monitor.start(interval)
self.publish(stats_monitor_set_topic, interval, options=PublishOptions(exclude=details.caller))
else:
emsg = "Cannot setup process statistics monitor: required packages not installed"
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
@wamp.register(None)
def trigger_gc(self, details=None):
"""
Manually trigger a garbage collection in this native process.
This procedure is registered under
``crossbar.node.<node_id>.worker.<worker_id>.trigger_gc``
for native workers and under
``crossbar.node.<node_id>.controller.trigger_gc``
for node controllers.
The procedure will publish an event when the garabage collection has finished to
``crossbar.node.<node_id>.worker.<worker_id>.on_gc_finished``
for native workers and
``crossbar.node.<node_id>.controller.on_gc_finished``
for node controllers:
.. code-block:: javascript
{
"requester": {
"session_id": 982734923,
"auth_id": "bob",
"auth_role": "admin"
},
"duration": 190
}
.. note:: The caller of this procedure will NOT receive the event.
:returns: Time (wall clock) consumed for garbage collection in ms.
:rtype: int
"""
self.log.debug("{cls}.trigger_gc", cls=self.__class__.__name__)
started = rtime()
# now trigger GC .. this is blocking!
gc.collect()
duration = int(round(1000. * (rtime() - started)))
on_gc_finished = u'{}.on_gc_finished'.format(self._uri_prefix)
self.publish(
on_gc_finished,
{
u'requester': {
u'session_id': details.caller,
# FIXME:
u'auth_id': None,
u'auth_role': None
},
u'duration': duration
},
options=PublishOptions(exclude=details.caller)
)
return duration
@wamp.register(None)
@inlineCallbacks
def start_manhole(self, config, details=None):
"""
Start a Manhole service within this process.
**Usage:**
This procedure is registered under
* ``crossbar.node.<node_id>.worker.<worker_id>.start_manhole`` - for native workers
* ``crossbar.node.<node_id>.controller.start_manhole`` - for node controllers
The procedure takes a Manhole service configuration which defines
a listening endpoint for the service and a list of users including
passwords, e.g.
.. code-block:: javascript
{
"endpoint": {
"type": "tcp",
"port": 6022
},
"users": [
{
"user": "******",
"password": "******"
}
]
}
**Errors:**
The procedure may raise the following errors:
* ``crossbar.error.invalid_configuration`` - the provided configuration is invalid
* ``crossbar.error.already_started`` - the Manhole service is already running (or starting)
* ``crossbar.error.feature_unavailable`` - the required support packages are not installed
**Events:**
The procedure will publish an event when the service **is starting** to
* ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_starting`` - for native workers
* ``crossbar.node.<node_id>.controller.on_manhole_starting`` - for node controllers
and publish an event when the service **has started** to
* ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_started`` - for native workers
* ``crossbar.node.<node_id>.controller.on_manhole_started`` - for node controllers
:param config: Manhole service configuration.
:type config: dict
"""
self.log.debug("{cls}.start_manhole(config = {config})",
cls=self.__class__.__name__, config=config)
if not _HAS_MANHOLE:
emsg = "Could not start manhole: required packages are missing ({})".format(_MANHOLE_MISSING_REASON)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
if self._manhole_service:
emsg = "Could not start manhole - already running (or starting)"
self.log.warn(emsg)
raise ApplicationError(u"crossbar.error.already_started", emsg)
try:
self.personality.check_manhole(self.personality, config)
except Exception as e:
emsg = "Could not start manhole: invalid configuration ({})".format(e)
self.log.error(emsg)
raise ApplicationError(u'crossbar.error.invalid_configuration', emsg)
from twisted.conch.ssh import keys
from twisted.conch.manhole_ssh import (
ConchFactory, TerminalRealm, TerminalSession)
from twisted.conch.manhole import ColoredManhole
from twisted.conch.checkers import SSHPublicKeyDatabase
class PublicKeyChecker(SSHPublicKeyDatabase):
def __init__(self, userKeys):
self.userKeys = {}
for username, keyData in userKeys.items():
self.userKeys[username] = keys.Key.fromString(data=keyData).blob()
def checkKey(self, credentials):
username = credentials.username.decode('utf8')
if username in self.userKeys:
keyBlob = self.userKeys[username]
return keyBlob == credentials.blob
# setup user authentication
#
authorized_keys = {
'oberstet': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCz7K1QwDhaq/Bi8o0uqiJQuVFCDQL5rbRvMClLHRx9KE3xP2Fh2eapzXuYGSgtG9Fyz1UQd+1oNM3wuNnT/DsBUBQrECP4bpFIHcJkMaFTARlCagkXosWsadzNnkW0osUCuHYMrzBJuXWF2GH+0OFCtVu+8E+4Mhvchu9xsHG8PM92SpI6aP0TtmT9D/0Bsm9JniRj8kndeS+iWG4s/pEGj7Rg7eGnbyQJt/9Jc1nWl6PngGbwp63dMVmh+8LP49PtfnxY8m9fdwpL4oW9U8beYqm8hyfBPN2yDXaehg6RILjIa7LU2/6bu96ZgnIz26zi/X9XlnJQt2aahWJs1+GR oberstet@thinkpad-t430s'
}
checker = PublicKeyChecker(authorized_keys)
# setup manhole namespace
#
namespace = {'session': self}
class PatchedTerminalSession(TerminalSession):
# get rid of
# exceptions.AttributeError: TerminalSession instance has no attribute 'windowChanged'
def windowChanged(self, winSize):
pass
rlm = TerminalRealm()
rlm.sessionFactory = PatchedTerminalSession # monkey patch
rlm.chainedProtocolFactory.protocolFactory = lambda _: ColoredManhole(namespace)
ptl = portal.Portal(rlm)
ptl.registerChecker(checker)
factory = ConchFactory(ptl)
factory.noisy = False
private_key = keys.Key.fromFile(os.path.join(self.cbdir, 'ssh_host_rsa_key'))
public_key = private_key.public()
publicKeys = {
b'ssh-rsa': public_key
}
privateKeys = {
b'ssh-rsa': private_key
}
factory.publicKeys = publicKeys
factory.privateKeys = privateKeys
self._manhole_service = ManholeService(config, details.caller)
starting_topic = '{}.on_manhole_starting'.format(self._uri_prefix)
starting_info = self._manhole_service.marshal()
# the caller gets a progressive result ..
if details.progress:
details.progress(starting_info)
# .. while all others get an event
self.publish(starting_topic, starting_info, options=PublishOptions(exclude=details.caller))
try:
self._manhole_service.port = yield create_listening_port_from_config(config['endpoint'],
self.cbdir,
factory,
self._reactor,
self.log)
except Exception as e:
self._manhole_service = None
emsg = "Manhole service endpoint cannot listen: {}".format(e)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.cannot_listen", emsg)
# alright, manhole has started
self._manhole_service.started = datetime.utcnow()
self._manhole_service.status = 'started'
started_topic = '{}.on_manhole_started'.format(self._uri_prefix)
started_info = self._manhole_service.marshal()
self.publish(started_topic, started_info, options=PublishOptions(exclude=details.caller))
returnValue(started_info)
@wamp.register(None)
@inlineCallbacks
def stop_manhole(self, details=None):
"""
Stop the Manhole service running in this process.
This procedure is registered under
* ``crossbar.node.<node_id>.worker.<worker_id>.stop_manhole`` for native workers and under
* ``crossbar.node.<node_id>.controller.stop_manhole`` for node controllers
When no Manhole service is currently running within this process,
or the Manhole service is already shutting down, a
``crossbar.error.not_started`` WAMP error is raised.
The procedure will publish an event when the service **is stopping** to
* ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_stopping`` for native workers and
* ``crossbar.node.<node_id>.controller.on_manhole_stopping`` for node controllers
and will publish an event when the service **has stopped** to
* ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_stopped`` for native workers and
* ``crossbar.node.<node_id>.controller.on_manhole_stopped`` for node controllers
"""
self.log.debug("{cls}.stop_manhole", cls=self.__class__.__name__)
if not self._manhole_service or self._manhole_service.status != 'started':
emsg = "Cannot stop manhole: not running (or already shutting down)"
raise ApplicationError(u"crossbar.error.not_started", emsg)
self._manhole_service.status = 'stopping'
stopping_topic = u'{}.on_manhole_stopping'.format(self._uri_prefix)
stopping_info = None
# the caller gets a progressive result ..
if details.progress:
details.progress(stopping_info)
# .. while all others get an event
self.publish(stopping_topic, stopping_info, options=PublishOptions(exclude=details.caller))
try:
yield self._manhole_service.port.stopListening()
except Exception as e:
self.log.warn("error while stop listening on endpoint: {error}", error=e)
self._manhole_service = None
stopped_topic = u'{}.on_manhole_stopped'.format(self._uri_prefix)
stopped_info = None
self.publish(stopped_topic, stopped_info, options=PublishOptions(exclude=details.caller))
returnValue(stopped_info)
@wamp.register(None)
def get_manhole(self, details=None):
"""
Get current manhole service information.
:returns: A dict with service information or `None` if the service is not running.
"""
self.log.debug("{cls}.get_manhole", cls=self.__class__.__name__)
if not _HAS_MANHOLE:
emsg = "Could not start manhole: required packages are missing ({})".format(_MANHOLE_MISSING_REASON)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
if not self._manhole_service:
return None
else:
return self._manhole_service.marshal()
@wamp.register(None)
def utcnow(self, details=None):
"""
Return current time as determined from within this process.
**Usage:**
This procedure is registered under
* ``crossbar.node.<node_id>.worker.<worker_id>.utcnow`` for native workers and under
* ``crossbar.node.<node_id>.controller.utcnow`` for node controllers
:returns: Current time (UTC) in UTC ISO 8601 format.
:rtype: str
"""
self.log.debug("{cls}.utcnow", cls=self.__class__.__name__)
return utcnow()
@wamp.register(None)
def started(self, details=None):
"""
Return start time of this process.
**Usage:**
This procedure is registered under
* ``crossbar.node.<node_id>.worker.<worker_id>.started`` for native workers and under
* ``crossbar.node.<node_id>.controller.started`` for node controllers
:returns: Start time (UTC) in UTC ISO 8601 format.
:rtype: str
"""
self.log.debug("{cls}.started", cls=self.__class__.__name__)
return utcstr(self._started)
@wamp.register(None)
def uptime(self, details=None):
"""
Return uptime of this process.
**Usage:**
This procedure is registered under
* ``crossbar.node.<node_id>.worker.<worker_id>.uptime`` for native workers and under
* ``crossbar.node.<node_id>.controller.uptime`` for node controllers
:returns: Uptime in seconds.
:rtype: float
"""
self.log.debug("{cls}.uptime", cls=self.__class__.__name__)
now = datetime.utcnow()
return (now - self._started).total_seconds()
class NativeProcess(ApplicationSession):
"""
A native Crossbar.io process (currently: controller, router or container).
"""
log = make_logger()
def __init__(self, config=None, reactor=None, personality=None):
# Twisted reactor
if not reactor:
from twisted.internet import reactor
self._reactor = reactor
self._reactor = reactor
# node/software personality
if personality:
self.personality = personality
else:
from crossbar.personality import Personality
self.personality = Personality
# base ctor
super(ApplicationSession, self).__init__(config=config)
def onConnect(self, do_join=True):
"""
"""
if not hasattr(self, 'cbdir'):
self.cbdir = self.config.extra.cbdir
if not hasattr(self, '_uri_prefix'):
self._uri_prefix = 'crossbar.node.{}'.format(
self.config.extra.node)
self._started = datetime.utcnow()
# see: BaseSession
self.include_traceback = False
self._manhole_service = None
if _HAS_PSUTIL:
self._pinfo = ProcessInfo()
self._pinfo_monitor = None
self._pinfo_monitor_seq = 0
else:
self._pinfo = None
self._pinfo_monitor = None
self._pinfo_monitor_seq = None
self.log.info("Process utilities not available")
self._connections = {}
if do_join:
self.join(self.config.realm)
@inlineCallbacks
def onJoin(self, details):
"""
Called when process has joined the node's management realm.
"""
regs = yield self.register(
self,
prefix=u'{}.'.format(self._uri_prefix),
options=RegisterOptions(details_arg='details'),
)
self.log.info("Registered {len_reg} procedures", len_reg=len(regs))
for reg in regs:
if isinstance(reg, Failure):
self.log.error("Failed to register: {f}",
f=reg,
log_failure=reg)
else:
self.log.debug(' {proc}', proc=reg.procedure)
returnValue(regs)
@wamp.register(None)
def get_cpu_count(self, logical=True, details=None):
"""
Returns the CPU core count on the machine this process is running on.
:param logical: If enabled (default), include logical CPU cores ("Hyperthreading"),
else only count physical CPU cores.
:type logical: bool
:returns: The number of CPU cores.
:rtype: int
"""
if not _HAS_PSUTIL:
emsg = "unable to get CPU count: required package 'psutil' is not installed"
self.log.warn(emsg)
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
return psutil.cpu_count(logical=logical)
@wamp.register(None)
def get_cpus(self, details=None):
"""
:returns: List of CPU IDs.
:rtype: list[Int]
"""
if not _HAS_PSUTIL:
emsg = "unable to get CPUs: required package 'psutil' is not installed"
self.log.warn(emsg)
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
return self._pinfo.cpus
@wamp.register(None)
def get_cpu_affinity(self, details=None):
"""
Get CPU affinity of this process.
:returns: List of CPU IDs the process affinity is set to.
:rtype: list of int
"""
if not _HAS_PSUTIL:
emsg = "unable to get CPU affinity: required package 'psutil' is not installed"
self.log.warn(emsg)
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
try:
p = psutil.Process(os.getpid())
current_affinity = p.cpu_affinity()
except Exception as e:
emsg = "Could not get CPU affinity: {}".format(e)
self.log.failure(emsg)
raise ApplicationError(u"crossbar.error.runtime_error", emsg)
else:
return current_affinity
@wamp.register(None)
def set_cpu_affinity(self, cpus, relative=True, details=None):
"""
Set CPU affinity of this process.
:param cpus: List of CPU IDs to set process affinity to. Each CPU ID must be
from the list `[0 .. N_CPUs]`, where N_CPUs can be retrieved via
``crossbar.worker.<worker_id>.get_cpu_count``.
:type cpus: list of int
:returns: List of CPU IDs the process affinity is set to.
:rtype: list of int
"""
if not _HAS_PSUTIL:
emsg = "Unable to set CPU affinity: required package 'psutil' is not installed"
self.log.warn(emsg)
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
if relative:
_cpu_ids = self._pinfo.cpus
_cpus = [_cpu_ids[i] for i in cpus]
else:
_cpus = cpus
try:
p = psutil.Process(os.getpid())
p.cpu_affinity(_cpus)
new_affinity = p.cpu_affinity()
if set(_cpus) != set(new_affinity):
raise Exception(
'CPUs mismatch after affinity setting ({} != {})'.format(
set(_cpus), set(new_affinity)))
except Exception as e:
emsg = "Could not set CPU affinity: {}".format(e)
self.log.failure(emsg)
raise ApplicationError(u"crossbar.error.runtime_error", emsg)
else:
# publish info to all but the caller ..
#
cpu_affinity_set_topic = u'{}.on_cpu_affinity_set'.format(
self._uri_prefix)
cpu_affinity_set_info = {
u'cpus': cpus,
u'relative': relative,
u'affinity': new_affinity,
u'who': details.caller
}
self.publish(cpu_affinity_set_topic,
cpu_affinity_set_info,
options=PublishOptions(exclude=details.caller))
# .. and return info directly to caller
#
return new_affinity
@inlineCallbacks
def start_connection(self, id, config, details=None):
"""
Starts a connection in this process.
:param id: The ID for the started connection.
:type id: unicode
:param config: Connection configuration.
:type config: dict
:param details: Caller details.
:type details: instance of :class:`autobahn.wamp.types.CallDetails`
:returns dict -- The connection.
"""
self.log.debug("start_connection: id={id}, config={config}",
id=id,
config=config)
# prohibit starting a component twice
#
if id in self._connections:
emsg = "cannot start connection: a connection with id={} is already started".format(
id)
self.log.warn(emsg)
raise ApplicationError(u"crossbar.error.invalid_configuration",
emsg)
# check configuration
#
try:
self.personality.check_connection(self.personality, config)
except Exception as e:
emsg = "invalid connection configuration ({})".format(e)
self.log.warn(emsg)
raise ApplicationError(u"crossbar.error.invalid_configuration",
emsg)
else:
self.log.info("Starting {ptype} in process.", ptype=config['type'])
if config['type'] == u'postgresql.connection':
if _HAS_POSTGRESQL:
connection = PostgreSQLConnection(id, config)
else:
emsg = "unable to start connection - required PostgreSQL driver package not installed"
self.log.warn(emsg)
raise ApplicationError(u"crossbar.error.feature_unavailable",
emsg)
else:
# should not arrive here
raise Exception("logic error")
self._connections[id] = connection
try:
yield connection.start()
self.log.info(
"Connection {connection_type} started '{connection_id}'",
connection_id=id,
connection_type=config['type'])
except Exception as e:
del self._connections[id]
raise
state = connection.marshal()
self.publish(u'crossbar.node.process.on_connection_start', state)
returnValue(state)
@inlineCallbacks
def stop_connection(self, id, details=None):
"""
Stop a connection currently running within this process.
:param id: The ID of the connection to stop.
:type id: unicode
:param details: Caller details.
:type details: instance of :class:`autobahn.wamp.types.CallDetails`
:returns dict -- A dict with component start information.
"""
self.log.debug("stop_connection: id={id}", id=id)
if id not in self._connections:
raise ApplicationError(
u'crossbar.error.no_such_object',
'no connection with ID {} running in this process'.format(id))
connection = self._connections[id]
try:
yield connection.stop()
except Exception as e:
self.log.warn('could not stop connection {id}: {error}', error=e)
raise
del self._connections[id]
state = connection.marshal()
self.publish(u'crossbar.node.process.on_connection_stop', state)
returnValue(state)
def get_connections(self, details=None):
"""
Get connections currently running within this processs.
:param details: Caller details.
:type details: instance of :class:`autobahn.wamp.types.CallDetails`
:returns list -- List of connections.
"""
self.log.debug("get_connections")
res = []
for c in self._connections.values():
res.append(c.marshal())
return res
@wamp.register(None)
def get_process_info(self, details=None):
"""
Get process information (open files, sockets, ...).
:returns: dict -- Dictionary with process information.
"""
self.log.debug("{cls}.get_process_info", cls=self.__class__.__name__)
if self._pinfo:
# psutil.AccessDenied
# PermissionError: [Errno 13] Permission denied: '/proc/14787/io'
return self._pinfo.get_info()
else:
emsg = "Could not retrieve process statistics: required packages not installed"
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
@wamp.register(None)
def get_process_stats(self, details=None):
"""
Get process statistics (CPU, memory, I/O).
:returns: dict -- Dictionary with process statistics.
"""
self.log.debug("{cls}.get_process_stats", cls=self.__class__.__name__)
if self._pinfo:
return self._pinfo.get_stats()
else:
emsg = "Could not retrieve process statistics: required packages not installed"
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
@wamp.register(None)
def set_process_stats_monitoring(self, interval, details=None):
"""
Enable/disable periodic publication of process statistics.
:param interval: The monitoring interval in seconds. Set to 0 to disable monitoring.
:type interval: float
"""
self.log.debug(
"{cls}.set_process_stats_monitoring(interval = {interval})",
cls=self.__class__.__name__,
interval=interval)
if self._pinfo:
stats_monitor_set_topic = '{}.on_process_stats_monitoring_set'.format(
self._uri_prefix)
# stop and remove any existing monitor
if self._pinfo_monitor:
self._pinfo_monitor.stop()
self._pinfo_monitor = None
self.publish(stats_monitor_set_topic,
0,
options=PublishOptions(exclude=details.caller))
# possibly start a new monitor
if interval > 0:
stats_topic = '{}.on_process_stats'.format(self._uri_prefix)
def publish_stats():
stats = self._pinfo.get_stats()
self._pinfo_monitor_seq += 1
stats[u'seq'] = self._pinfo_monitor_seq
self.publish(stats_topic, stats)
self._pinfo_monitor = LoopingCall(publish_stats)
self._pinfo_monitor.start(interval)
self.publish(stats_monitor_set_topic,
interval,
options=PublishOptions(exclude=details.caller))
else:
emsg = "Cannot setup process statistics monitor: required packages not installed"
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
@wamp.register(None)
def trigger_gc(self, details=None):
"""
Manually trigger a garbage collection in this native process.
This procedure is registered under
``crossbar.node.<node_id>.worker.<worker_id>.trigger_gc``
for native workers and under
``crossbar.node.<node_id>.controller.trigger_gc``
for node controllers.
The procedure will publish an event when the garabage collection has finished to
``crossbar.node.<node_id>.worker.<worker_id>.on_gc_finished``
for native workers and
``crossbar.node.<node_id>.controller.on_gc_finished``
for node controllers:
.. code-block:: javascript
{
"requester": {
"session_id": 982734923,
"auth_id": "bob",
"auth_role": "admin"
},
"duration": 190
}
.. note:: The caller of this procedure will NOT receive the event.
:returns: Time (wall clock) consumed for garbage collection in ms.
:rtype: int
"""
self.log.debug("{cls}.trigger_gc", cls=self.__class__.__name__)
started = rtime()
# now trigger GC .. this is blocking!
gc.collect()
duration = int(round(1000. * (rtime() - started)))
on_gc_finished = u'{}.on_gc_finished'.format(self._uri_prefix)
self.publish(
on_gc_finished,
{
u'requester': {
u'session_id': details.caller,
# FIXME:
u'auth_id': None,
u'auth_role': None
},
u'duration': duration
},
options=PublishOptions(exclude=details.caller))
return duration
@wamp.register(None)
@inlineCallbacks
def start_manhole(self, config, details=None):
"""
Start a Manhole service within this process.
**Usage:**
This procedure is registered under
* ``crossbar.node.<node_id>.worker.<worker_id>.start_manhole`` - for native workers
* ``crossbar.node.<node_id>.controller.start_manhole`` - for node controllers
The procedure takes a Manhole service configuration which defines
a listening endpoint for the service and a list of users including
passwords, e.g.
.. code-block:: javascript
{
"endpoint": {
"type": "tcp",
"port": 6022
},
"users": [
{
"user": "******",
"password": "******"
}
]
}
**Errors:**
The procedure may raise the following errors:
* ``crossbar.error.invalid_configuration`` - the provided configuration is invalid
* ``crossbar.error.already_started`` - the Manhole service is already running (or starting)
* ``crossbar.error.feature_unavailable`` - the required support packages are not installed
**Events:**
The procedure will publish an event when the service **is starting** to
* ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_starting`` - for native workers
* ``crossbar.node.<node_id>.controller.on_manhole_starting`` - for node controllers
and publish an event when the service **has started** to
* ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_started`` - for native workers
* ``crossbar.node.<node_id>.controller.on_manhole_started`` - for node controllers
:param config: Manhole service configuration.
:type config: dict
"""
self.log.debug("{cls}.start_manhole(config = {config})",
cls=self.__class__.__name__,
config=config)
if not _HAS_MANHOLE:
emsg = "Could not start manhole: required packages are missing ({})".format(
_MANHOLE_MISSING_REASON)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
if self._manhole_service:
emsg = "Could not start manhole - already running (or starting)"
self.log.warn(emsg)
raise ApplicationError(u"crossbar.error.already_started", emsg)
try:
self.personality.check_manhole(self.personality, config)
except Exception as e:
emsg = "Could not start manhole: invalid configuration ({})".format(
e)
self.log.error(emsg)
raise ApplicationError(u'crossbar.error.invalid_configuration',
emsg)
from twisted.conch.ssh import keys
from twisted.conch.manhole_ssh import (ConchFactory, TerminalRealm,
TerminalSession)
from twisted.conch.manhole import ColoredManhole
from twisted.conch.checkers import SSHPublicKeyDatabase
class PublicKeyChecker(SSHPublicKeyDatabase):
def __init__(self, userKeys):
self.userKeys = {}
for username, keyData in userKeys.items():
self.userKeys[username] = keys.Key.fromString(
data=keyData).blob()
def checkKey(self, credentials):
username = credentials.username.decode('utf8')
if username in self.userKeys:
keyBlob = self.userKeys[username]
return keyBlob == credentials.blob
# setup user authentication
#
authorized_keys = {
'oberstet':
'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCz7K1QwDhaq/Bi8o0uqiJQuVFCDQL5rbRvMClLHRx9KE3xP2Fh2eapzXuYGSgtG9Fyz1UQd+1oNM3wuNnT/DsBUBQrECP4bpFIHcJkMaFTARlCagkXosWsadzNnkW0osUCuHYMrzBJuXWF2GH+0OFCtVu+8E+4Mhvchu9xsHG8PM92SpI6aP0TtmT9D/0Bsm9JniRj8kndeS+iWG4s/pEGj7Rg7eGnbyQJt/9Jc1nWl6PngGbwp63dMVmh+8LP49PtfnxY8m9fdwpL4oW9U8beYqm8hyfBPN2yDXaehg6RILjIa7LU2/6bu96ZgnIz26zi/X9XlnJQt2aahWJs1+GR oberstet@thinkpad-t430s'
}
checker = PublicKeyChecker(authorized_keys)
# setup manhole namespace
#
namespace = {'session': self}
class PatchedTerminalSession(TerminalSession):
# get rid of
# exceptions.AttributeError: TerminalSession instance has no attribute 'windowChanged'
def windowChanged(self, winSize):
pass
rlm = TerminalRealm()
rlm.sessionFactory = PatchedTerminalSession # monkey patch
rlm.chainedProtocolFactory.protocolFactory = lambda _: ColoredManhole(
namespace)
ptl = portal.Portal(rlm)
ptl.registerChecker(checker)
factory = ConchFactory(ptl)
factory.noisy = False
private_key = keys.Key.fromFile(
os.path.join(self.cbdir, 'ssh_host_rsa_key'))
public_key = private_key.public()
publicKeys = {b'ssh-rsa': public_key}
privateKeys = {b'ssh-rsa': private_key}
factory.publicKeys = publicKeys
factory.privateKeys = privateKeys
self._manhole_service = ManholeService(config, details.caller)
starting_topic = '{}.on_manhole_starting'.format(self._uri_prefix)
starting_info = self._manhole_service.marshal()
# the caller gets a progressive result ..
if details.progress:
details.progress(starting_info)
# .. while all others get an event
self.publish(starting_topic,
starting_info,
options=PublishOptions(exclude=details.caller))
try:
self._manhole_service.port = yield create_listening_port_from_config(
config['endpoint'], self.cbdir, factory, self._reactor,
self.log)
except Exception as e:
self._manhole_service = None
emsg = "Manhole service endpoint cannot listen: {}".format(e)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.cannot_listen", emsg)
# alright, manhole has started
self._manhole_service.started = datetime.utcnow()
self._manhole_service.status = 'started'
started_topic = '{}.on_manhole_started'.format(self._uri_prefix)
started_info = self._manhole_service.marshal()
self.publish(started_topic,
started_info,
options=PublishOptions(exclude=details.caller))
returnValue(started_info)
@wamp.register(None)
@inlineCallbacks
def stop_manhole(self, details=None):
"""
Stop the Manhole service running in this process.
This procedure is registered under
* ``crossbar.node.<node_id>.worker.<worker_id>.stop_manhole`` for native workers and under
* ``crossbar.node.<node_id>.controller.stop_manhole`` for node controllers
When no Manhole service is currently running within this process,
or the Manhole service is already shutting down, a
``crossbar.error.not_started`` WAMP error is raised.
The procedure will publish an event when the service **is stopping** to
* ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_stopping`` for native workers and
* ``crossbar.node.<node_id>.controller.on_manhole_stopping`` for node controllers
and will publish an event when the service **has stopped** to
* ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_stopped`` for native workers and
* ``crossbar.node.<node_id>.controller.on_manhole_stopped`` for node controllers
"""
self.log.debug("{cls}.stop_manhole", cls=self.__class__.__name__)
if not self._manhole_service or self._manhole_service.status != 'started':
emsg = "Cannot stop manhole: not running (or already shutting down)"
raise ApplicationError(u"crossbar.error.not_started", emsg)
self._manhole_service.status = 'stopping'
stopping_topic = u'{}.on_manhole_stopping'.format(self._uri_prefix)
stopping_info = None
# the caller gets a progressive result ..
if details.progress:
details.progress(stopping_info)
# .. while all others get an event
self.publish(stopping_topic,
stopping_info,
options=PublishOptions(exclude=details.caller))
try:
yield self._manhole_service.port.stopListening()
except Exception as e:
self.log.warn("error while stop listening on endpoint: {error}",
error=e)
self._manhole_service = None
stopped_topic = u'{}.on_manhole_stopped'.format(self._uri_prefix)
stopped_info = None
self.publish(stopped_topic,
stopped_info,
options=PublishOptions(exclude=details.caller))
returnValue(stopped_info)
@wamp.register(None)
def get_manhole(self, details=None):
"""
Get current manhole service information.
:returns: dict -- A dict with service information or `None` if the service is not running.
"""
self.log.debug("{cls}.get_manhole", cls=self.__class__.__name__)
if not _HAS_MANHOLE:
emsg = "Could not start manhole: required packages are missing ({})".format(
_MANHOLE_MISSING_REASON)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
if not self._manhole_service:
return None
else:
return self._manhole_service.marshal()
@wamp.register(None)
def utcnow(self, details=None):
"""
Return current time as determined from within this process.
**Usage:**
This procedure is registered under
* ``crossbar.node.<node_id>.worker.<worker_id>.utcnow`` for native workers and under
* ``crossbar.node.<node_id>.controller.utcnow`` for node controllers
:returns: Current time (UTC) in UTC ISO 8601 format.
:rtype: unicode
"""
self.log.debug("{cls}.utcnow", cls=self.__class__.__name__)
return utcnow()
@wamp.register(None)
def started(self, details=None):
"""
Return start time of this process.
**Usage:**
This procedure is registered under
* ``crossbar.node.<node_id>.worker.<worker_id>.started`` for native workers and under
* ``crossbar.node.<node_id>.controller.started`` for node controllers
:returns: Start time (UTC) in UTC ISO 8601 format.
:rtype: unicode
"""
self.log.debug("{cls}.started", cls=self.__class__.__name__)
return utcstr(self._started)
@wamp.register(None)
def uptime(self, details=None):
"""
Return uptime of this process.
**Usage:**
This procedure is registered under
* ``crossbar.node.<node_id>.worker.<worker_id>.uptime`` for native workers and under
* ``crossbar.node.<node_id>.controller.uptime`` for node controllers
:returns: Uptime in seconds.
:rtype: float
"""
self.log.debug("{cls}.uptime", cls=self.__class__.__name__)
now = datetime.utcnow()
return (now - self._started).total_seconds()
class NativeProcessSession(ApplicationSession):
"""
A native Crossbar.io process (currently: controller, router or container).
"""
def onConnect(self, do_join=True):
"""
"""
if not hasattr(self, 'debug'):
self.debug = self.config.extra.debug
if not hasattr(self, 'cbdir'):
self.cbdir = self.config.extra.cbdir
if not hasattr(self, '_uri_prefix'):
self._uri_prefix = 'crossbar.node.{}'.format(
self.config.extra.node)
if self.debug:
log.msg("Session connected to management router")
self._started = datetime.utcnow()
# see: BaseSession
self.include_traceback = False
self.debug_app = False
self._manhole_service = None
if _HAS_PSUTIL:
self._pinfo = ProcessInfo()
self._pinfo_monitor = None
self._pinfo_monitor_seq = 0
else:
self._pinfo = None
self._pinfo_monitor = None
self._pinfo_monitor_seq = None
log.msg("Warning: process utilities not available")
if do_join:
self.join(self.config.realm)
@inlineCallbacks
def onJoin(self, details):
"""
Called when process has joined the node's management realm.
"""
procs = [
'start_manhole', 'stop_manhole', 'get_manhole', 'trigger_gc',
'utcnow', 'started', 'uptime', 'get_process_info',
'get_process_stats', 'set_process_stats_monitoring'
]
dl = []
for proc in procs:
uri = '{}.{}'.format(self._uri_prefix, proc)
if self.debug:
log.msg("Registering procedure '{}'".format(uri))
dl.append(
self.register(getattr(self, proc),
uri,
options=RegisterOptions(details_arg='details')))
regs = yield DeferredList(dl)
if self.debug:
log.msg("{} registered {} procedures".format(
self.__class__.__name__, len(regs)))
def get_process_info(self, details=None):
"""
Get process information (open files, sockets, ...).
:returns: dict -- Dictionary with process information.
"""
if self.debug:
log.msg("{}.get_process_info".format(self.__class__.__name__))
if self._pinfo:
return self._pinfo.get_info()
else:
emsg = "ERROR: could not retrieve process statistics - required packages not installed"
raise ApplicationError("crossbar.error.feature_unavailable", emsg)
def get_process_stats(self, details=None):
"""
Get process statistics (CPU, memory, I/O).
:returns: dict -- Dictionary with process statistics.
"""
if self.debug:
log.msg("{}.get_process_stats".format(self.__class__.__name__))
if self._pinfo:
return self._pinfo.get_stats()
else:
emsg = "ERROR: could not retrieve process statistics - required packages not installed"
raise ApplicationError("crossbar.error.feature_unavailable", emsg)
def set_process_stats_monitoring(self, interval, details=None):
"""
Enable/disable periodic publication of process statistics.
:param interval: The monitoring interval in seconds. Set to 0 to disable monitoring.
:type interval: float
"""
if self.debug:
log.msg(
"{}.set_process_stats_monitoring".format(
self.__class__.__name__), interval)
if self._pinfo:
stats_monitor_set_topic = '{}.on_process_stats_monitoring_set'.format(
self._uri_prefix)
# stop and remove any existing monitor
if self._pinfo_monitor:
self._pinfo_monitor.stop()
self._pinfo_monitor = None
self.publish(stats_monitor_set_topic,
0,
options=PublishOptions(exclude=[details.caller]))
# possibly start a new monitor
if interval > 0:
stats_topic = '{}.on_process_stats'.format(self._uri_prefix)
def publish_stats():
stats = self._pinfo.get_stats()
self._pinfo_monitor_seq += 1
stats['seq'] = self._pinfo_monitor_seq
self.publish(stats_topic, stats)
self._pinfo_monitor = LoopingCall(publish_stats)
self._pinfo_monitor.start(interval)
self.publish(stats_monitor_set_topic,
interval,
options=PublishOptions(exclude=[details.caller]))
else:
emsg = "ERROR: cannot setup process statistics monitor - required packages not installed"
raise ApplicationError("crossbar.error.feature_unavailable", emsg)
def trigger_gc(self, details=None):
"""
Triggers a garbage collection.
:returns: float -- Time consumed for GC in ms.
"""
if self.debug:
log.msg("{}.trigger_gc".format(self.__class__.__name__))
started = rtime()
gc.collect()
return 1000. * (rtime() - started)
@inlineCallbacks
def start_manhole(self, config, details=None):
"""
Start a manhole (SSH) within this worker.
:param config: Manhole configuration.
:type config: obj
"""
if self.debug:
log.msg("{}.start_manhole".format(self.__class__.__name__), config)
if not _HAS_MANHOLE:
emsg = "ERROR: could not start manhole - required packages are missing ({})".format(
_MANHOLE_MISSING_REASON)
log.msg(emsg)
raise ApplicationError("crossbar.error.feature_unavailable", emsg)
if self._manhole_service:
emsg = "ERROR: could not start manhole - already running (or starting)"
log.msg(emsg)
raise ApplicationError("crossbar.error.already_started", emsg)
try:
checkconfig.check_manhole(config)
except Exception as e:
emsg = "ERROR: could not start manhole - invalid configuration ({})".format(
e)
log.msg(emsg)
raise ApplicationError('crossbar.error.invalid_configuration',
emsg)
# setup user authentication
#
checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()
for user in config['users']:
checker.addUser(user['user'], user['password'])
# setup manhole namespace
#
namespace = {'session': self}
class PatchedTerminalSession(TerminalSession):
# get rid of
# exceptions.AttributeError: TerminalSession instance has no attribute 'windowChanged'
def windowChanged(self, winSize):
pass
rlm = TerminalRealm()
rlm.sessionFactory = PatchedTerminalSession # monkey patch
rlm.chainedProtocolFactory.protocolFactory = lambda _: ColoredManhole(
namespace)
ptl = portal.Portal(rlm, [checker])
factory = ConchFactory(ptl)
factory.noisy = False
self._manhole_service = ManholeService(config, details.caller)
starting_topic = '{}.on_manhole_starting'.format(self._uri_prefix)
starting_info = self._manhole_service.marshal()
# the caller gets a progressive result ..
if details.progress:
details.progress(starting_info)
# .. while all others get an event
self.publish(starting_topic,
starting_info,
options=PublishOptions(exclude=[details.caller]))
try:
self._manhole_service.port = yield create_listening_port_from_config(
config['endpoint'], factory, self.cbdir, reactor)
except Exception as e:
self._manhole_service = None
emsg = "ERROR: manhole service endpoint cannot listen - {}".format(
e)
log.msg(emsg)
raise ApplicationError("crossbar.error.cannot_listen", emsg)
# alright, manhole has started
self._manhole_service.started = datetime.utcnow()
self._manhole_service.status = 'started'
started_topic = '{}.on_manhole_started'.format(self._uri_prefix)
started_info = self._manhole_service.marshal()
self.publish(started_topic,
started_info,
options=PublishOptions(exclude=[details.caller]))
returnValue(started_info)
@inlineCallbacks
def stop_manhole(self, details=None):
"""
Stop Manhole.
"""
if self.debug:
log.msg("{}.stop_manhole".format(self.__class__.__name__))
if not _HAS_MANHOLE:
emsg = "ERROR: could not start manhole - required packages are missing ({})".format(
_MANHOLE_MISSING_REASON)
log.msg(emsg)
raise ApplicationError("crossbar.error.feature_unavailable", emsg)
if not self._manhole_service or self._manhole_service.status != 'started':
emsg = "ERROR: cannot stop manhole - not running (or already shutting down)"
raise ApplicationError("crossbar.error.not_started", emsg)
self._manhole_service.status = 'stopping'
stopping_topic = '{}.on_manhole_stopping'.format(self._uri_prefix)
stopping_info = None
# the caller gets a progressive result ..
if details.progress:
details.progress(stopping_info)
# .. while all others get an event
self.publish(stopping_topic,
stopping_info,
options=PublishOptions(exclude=[details.caller]))
try:
yield self._manhole_service.port.stopListening()
except Exception as e:
raise Exception(
"INTERNAL ERROR: don't know how to handle a failed called to stopListening() - {}"
.format(e))
self._manhole_service = None
stopped_topic = '{}.on_manhole_stopped'.format(self._uri_prefix)
stopped_info = None
self.publish(stopped_topic,
stopped_info,
options=PublishOptions(exclude=[details.caller]))
returnValue(stopped_info)
def get_manhole(self, details=None):
"""
Get current manhole service information.
:returns: dict -- A dict with service information or `None` if the service is not running.
"""
if self.debug:
log.msg("{}.get_manhole".format(self.__class__.__name__))
if not _HAS_MANHOLE:
emsg = "ERROR: could not start manhole - required packages are missing ({})".format(
_MANHOLE_MISSING_REASON)
log.msg(emsg)
raise ApplicationError("crossbar.error.feature_unavailable", emsg)
if not self._manhole_service:
return None
else:
return self._manhole_service.marshal()
def utcnow(self, details=None):
"""
Return current time as determined from within this process.
:returns str -- Current time (UTC) in UTC ISO 8601 format.
"""
if self.debug:
log.msg("{}.utcnow".format(self.__class__.__name__))
return utcnow()
def started(self, details=None):
"""
Return start time of this process.
:returns str -- Start time (UTC) in UTC ISO 8601 format.
"""
if self.debug:
log.msg("{}.started".format(self.__class__.__name__))
return utcstr(self._started)
def uptime(self, details=None):
"""
Uptime of this process.
:returns float -- Uptime in seconds.
"""
if self.debug:
log.msg("{}.uptime".format(self.__class__.__name__))
now = datetime.utcnow()
return (now - self._started).total_seconds()
class AppSession(ApplicationSession):
@inlineCallbacks
def onJoin(self, details):
self._db = Database(dbpath='../results')
self._schema = Schema.attach(self._db)
self._pinfo = ProcessInfo()
self._logname = self.config.extra['logname']
self._period = self.config.extra.get('period', 10.)
self._running = True
batch_id = uuid.uuid4()
self._last_stats = None
self._stats_loop = LoopingCall(self._stats, batch_id)
self._stats_loop.start(self._period)
dl = []
for i in range(8):
d = self._loop(batch_id, i)
dl.append(d)
d = gatherResults(dl)
try:
yield d
except TransportLost:
pass
def onLeave(self, details):
self._running = False
self._stats_loop.stop()
def _stats(self, batch_id):
stats = self._pinfo.get_stats()
if self._last_stats:
batch_duration = (stats['time'] - self._last_stats['time']) / 10**9
ctx = round((stats['voluntary'] - self._last_stats['voluntary']) /
batch_duration, 0)
self.log.info(
'{logprefix}: {user} user, {system} system, {mem_percent} mem_percent, {ctx} ctx',
logprefix=hl('LOAD {}.*'.format(self._logname),
color='magenta',
bold=True),
logname=self._logname,
user=stats['user'],
system=stats['system'],
mem_percent=round(stats['mem_percent'], 1),
ctx=ctx)
self._last_stats = stats
@inlineCallbacks
def _loop(self, batch_id, index):
prefix = self.config.extra['prefix']
while self._running:
rtts = []
self.log.debug(
'{logprefix} is starting new period ..',
logprefix=hl(' {}.{}'.format(self._logname, index),
color='magenta',
bold=True),
)
batch_started = time_ns()
while float(time_ns() - batch_started) / 10**9 < self._period:
ts_req = time_ns()
res = yield self.call('{}.echo'.format(prefix), ts_req)
ts_res = time_ns()
assert res == ts_req
rtt = ts_res - ts_req
rtts.append(rtt)
batch_duration = float(time_ns() - batch_started) / 10**9
rtts = sorted(rtts)
sr = WampStatsRecord()
sr.worker = int(self._logname.split('.')[1])
sr.loop = index
sr.count = len(rtts)
sr.calls_per_sec = int(round(sr.count / batch_duration, 0))
# all times here are in microseconds:
sr.avg_rtt = int(
round(1000000. * batch_duration / float(sr.count), 0))
sr.max_rtt = int(round(rtts[-1] / 1000, 0))
sr.q50_rtt = int(round(rtts[int(sr.count / 2.)] / 1000, 0))
sr.q99_rtt = int(round(rtts[int(-(sr.count / 100.))] / 1000, 0))
sr.q995_rtt = int(round(rtts[int(-(sr.count / 995.))] / 1000, 0))
try:
with self._db.begin(write=True) as txn:
self._schema.wamp_stats[txn, (
batch_id, np.datetime64(batch_started, 'ns'))] = sr
except:
self.log.failure()
return self.leave()
self.log.info(
"{logprefix}: {count} calls, {calls_per_sec}, round-trip time (microseconds): q50 {q50_rtt}, avg {avg_rtt}, {q99_rtt}, q995 {q995_rtt}, max {max_rtt}"
.format(logprefix=hl('WAMP {}.{}'.format(self._logname, index),
color='magenta',
bold=True),
count=sr.count,
calls_per_sec=hl('{} calls/second'.format(
sr.calls_per_sec),
bold=True),
q50_rtt=sr.q50_rtt,
avg_rtt=sr.avg_rtt,
q99_rtt=hl('q99 {}'.format(sr.q99_rtt), bold=True),
q995_rtt=sr.q995_rtt,
max_rtt=sr.max_rtt))
class HATestClientSession(ApplicationSession):
TEST_TOPIC = 'com.example.test1.binary'
TEST_PROC = 'com.example.proc1.binary'
log = txaio.make_logger()
def proc1(self, logname, url, loop, counter, payload, details=None):
fingerprint = hashlib.sha256(payload).digest()[:6]
self.log.info(
'{logprefix}: INVOCATION received on pid={pid} from {sender} -> loop={loop}, counter={counter}, len={payload_len}, fp={fp}, caller={caller}, caller_authid={caller_authid}, caller_authrole={caller_authrole}, forward_for={forward_for}',
logprefix=hl('WAMP {}:{}'.format(self._url, self._logname),
color='blue',
bold=True),
pid=hl(self._pid, color='blue', bold=True),
sender=hl('{}:{}'.format(url, logname), color='blue', bold=True),
loop=loop,
counter=counter,
procedure=details.procedure,
payload_len=len(payload),
fp=hl(binascii.b2a_hex(fingerprint).decode(),
color='blue',
bold=True),
caller=hl(details.caller, color='blue', bold=True),
caller_authid=hl(details.caller_authid, color='blue', bold=True),
caller_authrole=hl(details.caller_authrole,
color='blue',
bold=True),
forward_for=details.forward_for)
fingerprint = hashlib.sha256(payload).digest()[:6]
self._received_calls_cnt += 1
self._received_calls_bytes += len(payload)
if details.forward_for:
self._received_calls_ff_cnt += 1
return fingerprint, self._pid, self._logname, self._url
def on_event1(self, logname, url, loop, counter, payload, details=None):
fingerprint = hashlib.sha256(payload).digest()[:6]
self.log.debug(
'{logprefix}: EVENT received from {sender} -> loop={loop}, counter={counter}, len={payload_len}, fp={fp}, publisher={publisher}, publisher_authid={publisher_authid}, publisher_authole={publisher_authrole}, forward_for={forward_for}',
logprefix=hl('WAMP {}:{}'.format(self._url, self._logname),
color='blue',
bold=True),
sender=hl('{}:{}'.format(url, logname), color='blue', bold=True),
loop=loop,
counter=counter,
topic=details.topic,
payload_len=len(payload),
fp=hl(binascii.b2a_hex(fingerprint).decode(),
color='blue',
bold=True),
publisher=hl(details.publisher, color='blue', bold=True),
publisher_authid=hl(details.publisher_authid,
color='blue',
bold=True),
publisher_authrole=hl(details.publisher_authrole,
color='blue',
bold=True),
forward_for=details.forward_for)
self._received_cnt += 1
self._received_bytes += len(payload)
if details.forward_for:
self._received_ff_cnt += 1
@inlineCallbacks
def onJoin(self, details):
self._pid = os.getpid()
# benchmark parametrization:
self._period = self.config.extra.get('period', 10)
self._loops = self.config.extra.get('loops', 1)
self._rate = self.config.extra.get('rate', 1)
self._stride = self.config.extra.get('stride', 1)
self._size = self.config.extra.get('size', 256)
self._logname = self.config.extra.get('logname', None)
self._url = self.config.extra.get('url', None)
self._silent = self.config.extra.get('silent', False)
self._batch_id = uuid.uuid4()
self._running = True
self._pinfo = ProcessInfo()
# run-time session statistics for EVENTs
self._received_cnt = 0
self._received_bytes = 0
self._received_ff_cnt = 0
self._published_cnt = 0
self._published_bytes = 0
# run-time session statistics for CALLs
self._received_calls_cnt = 0
self._received_calls_bytes = 0
self._received_calls_ff_cnt = 0
self._calls_cnt = 0
self._calls_bytes = 0
self.log.info('{logname} connected [batch="{batch_id}"]: {details}',
logname=self._logname,
batch_id=hl(self._batch_id),
details=details)
self._last_stats = None
self._stats_loop = None
if self._silent:
self._stats(self._batch_id, self.log.info)
stats_period = 10.
if stats_period:
self._stats_loop = LoopingCall(self._stats, self._batch_id,
self.log.info)
self._stats_loop.start(stats_period)
yield self.subscribe(self.on_event1,
HATestClientSession.TEST_TOPIC,
options=SubscribeOptions(match='exact',
details=True))
for i in range(self._loops):
self._sender_loop('{}.{}'.format(self._logname, i))
self.log.info(
'{logname} ready [period={period}, loops={loops}, rate={rate}, stride={stride}, size={size}]',
logname=self._logname,
period=self._period,
loops=self._loops,
rate=self._rate,
stride=self._stride,
size=self._size)
def onLeave(self, details):
self.log.info('{logname} leaving: reason="{reason}"',
logname=self._logname,
reason=details.reason)
self._running = False
if self._silent:
self._stats(self._batch_id, self.log.warn)
if self._stats_loop:
self._stats_loop.stop()
self._stats_loop = None
@inlineCallbacks
def _sender_loop(self, loopname, enable_publish=True, enable_call=True):
loop = 0
while self._running:
started = time_ns()
dl = []
for counter in range(self._stride):
payload = os.urandom(self._size)
fingerprint = hashlib.sha256(payload).digest()[:6]
if enable_publish:
d = self.publish(HATestClientSession.TEST_TOPIC,
self._logname,
self._url,
loop,
counter,
payload,
options=PublishOptions(acknowledge=True,
exclude_me=False))
dl.append(d)
self._published_cnt += 1
self._published_bytes += len(payload)
self.log.debug(
'{logprefix}: EVENT sent from session={session}, authid={authid}, authrole={authrole} -> loop={loop}, counter={counter}, len={payload_len}, fp={fp}',
logprefix=hl('WAMP {}:{}'.format(
self._url, self._logname),
color='green',
bold=True),
session=hl(self._session_id, color='green', bold=True),
authid=hl(self._authid, color='green', bold=True),
authrole=hl(self._authrole, color='green', bold=True),
loop=loop,
counter=counter,
topic=HATestClientSession.TEST_TOPIC,
payload_len=len(payload),
fp=hl(binascii.b2a_hex(fingerprint).decode(),
color='green',
bold=True))
if enable_call:
for uri in [
'node{}.container1.proc1'.format(i + 1)
for i in range(4)
]:
d = self.call(uri,
self._logname,
self._url,
loop,
counter,
payload,
options=CallOptions(details=True))
def check_result(result, uri):
print('-' * 100, result)
_fingerprint, _pid, _logname, _url = result.results[
0]
self.log.info(
'{logprefix}: CALL RESULT for {uri} received from pid={pid}, logname={logname}, url={url}, callee={callee}, callee_authid={callee_authid}, callee_authrole={callee_authrole}, forward_for={forward_for}, fp={fp} => fp_equal={fp_equal}',
logprefix=hl('WAMP {}:{}'.format(
self._url, self._logname),
color='green',
bold=True),
pid=hl(_pid, color='green', bold=True),
logname=_logname,
url=_url,
fp=hl(binascii.b2a_hex(fingerprint).decode(),
color='green',
bold=True),
fp_equal=(_fingerprint == fingerprint),
uri=hl(uri, color='yellow', bold=True),
callee=result.callee,
callee_authid=result.callee_authid,
callee_authrole=result.callee_authrole,
forward_for=result.forward_for)
assert _fingerprint == fingerprint
def error(err):
print(err)
d.addCallbacks(check_result, error, (uri, ))
dl.append(d)
self._calls_cnt += 1
self._calls_bytes += len(payload)
self.log.info(
'{logprefix}: CALL issued to {uri} from pid={pid}, session={session}, authid={authid}, authrole={authrole} -> loop={loop}, counter={counter}, len={payload_len}, fp={fp}',
logprefix=hl('WAMP {}:{}'.format(
self._url, self._logname),
color='green',
bold=True),
pid=hl(self._pid, color='green', bold=True),
session=hl(self._session_id,
color='green',
bold=True),
authid=hl(self._authid, color='green', bold=True),
authrole=hl(self._authrole,
color='green',
bold=True),
uri=hl(uri, color='yellow', bold=True),
loop=loop,
counter=counter,
procedure=HATestClientSession.TEST_PROC,
payload_len=len(payload),
fp=hl(binascii.b2a_hex(fingerprint).decode(),
color='green',
bold=True))
d = gatherResults(dl)
try:
yield d
except TransportLost:
self.log.error('Transport lost!')
self.leave()
return
duration = (time_ns() - started) / 10**9
sleep_secs = (1 / float(self._rate)) - duration
if sleep_secs > 0:
yield sleep(sleep_secs)
loop += 1
def _stats(self, batch_id, log):
stats = self._pinfo.get_stats()
if self._last_stats:
batch_duration = (stats['time'] - self._last_stats['time']) / 10**9
ctx = round((stats['voluntary'] - self._last_stats['voluntary']) /
batch_duration, 0)
log('{logprefix}: {user} user, {system} system, {mem_percent} mem_percent, {ctx} ctx',
logprefix=hl('LOAD', color='white', bold=True),
user=stats['user'],
system=stats['system'],
mem_percent=round(stats['mem_percent'], 1),
ctx=ctx)
events_received_per_sec = int(
round(self._received_cnt / batch_duration))
bytes_received_per_sec = int(
round(self._received_bytes / batch_duration))
events_published_per_sec = int(
round(self._published_cnt / batch_duration))
bytes_published_per_sec = int(
round(self._published_bytes / batch_duration))
log('{logprefix}: {events_received} EVENTs received ({events_received_ff} forwarded), {events_received_per_sec}, {events_published} events published, {events_published_per_sec} events/second',
logprefix=hl('WAMP {}.*'.format(self._logname),
color='white',
bold=True),
events_received=self._received_cnt,
events_received_ff=self._received_ff_cnt,
events_received_per_sec=hl(
'{} events/second'.format(events_received_per_sec),
color='white',
bold=True),
bytes_received_per_sec=bytes_received_per_sec,
events_published=self._published_cnt,
events_published_per_sec=events_published_per_sec,
bytes_published_per_sec=bytes_published_per_sec)
calls_received_per_sec = int(
round(self._received_calls_cnt / batch_duration))
call_bytes_received_per_sec = int(
round(self._received_calls_bytes / batch_duration))
calls_issued_per_sec = int(round(self._calls_cnt / batch_duration))
call_bytes_issued_per_sec = int(
round(self._calls_bytes / batch_duration))
log('{logprefix}: {calls_received} INVOCATIONs received ({calls_received_ff} forwarded), {calls_received_per_sec}, {calls_issued} calls issued, {calls_issued_per_sec} calls/second',
logprefix=hl('WAMP {}.*'.format(self._logname),
color='white',
bold=True),
calls_received=self._received_calls_cnt,
calls_received_ff=self._received_calls_ff_cnt,
calls_received_per_sec=hl(
'{} calls/second'.format(calls_received_per_sec),
color='white',
bold=True),
call_bytes_received_per_sec=call_bytes_received_per_sec,
calls_issued=self._calls_cnt,
calls_issued_per_sec=calls_issued_per_sec,
call_bytes_issued_per_sec=call_bytes_issued_per_sec)
self._received_cnt = 0
self._received_bytes = 0
self._received_ff_cnt = 0
self._published_cnt = 0
self._published_bytes = 0
self._received_calls_cnt = 0
self._received_calls_bytes = 0
self._received_calls_ff_cnt = 0
self._calls_cnt = 0
self._calls_bytes = 0
self._last_stats = stats
def onJoin(self, details):
self._pid = os.getpid()
# benchmark parametrization:
self._period = self.config.extra.get('period', 10)
self._loops = self.config.extra.get('loops', 1)
self._rate = self.config.extra.get('rate', 1)
self._stride = self.config.extra.get('stride', 1)
self._size = self.config.extra.get('size', 256)
self._logname = self.config.extra.get('logname', None)
self._url = self.config.extra.get('url', None)
self._silent = self.config.extra.get('silent', False)
self._batch_id = uuid.uuid4()
self._running = True
self._pinfo = ProcessInfo()
# run-time session statistics for EVENTs
self._received_cnt = 0
self._received_bytes = 0
self._received_ff_cnt = 0
self._published_cnt = 0
self._published_bytes = 0
# run-time session statistics for CALLs
self._received_calls_cnt = 0
self._received_calls_bytes = 0
self._received_calls_ff_cnt = 0
self._calls_cnt = 0
self._calls_bytes = 0
self.log.info('{logname} connected [batch="{batch_id}"]: {details}',
logname=self._logname,
batch_id=hl(self._batch_id),
details=details)
self._last_stats = None
self._stats_loop = None
if self._silent:
self._stats(self._batch_id, self.log.info)
stats_period = 10.
if stats_period:
self._stats_loop = LoopingCall(self._stats, self._batch_id,
self.log.info)
self._stats_loop.start(stats_period)
yield self.subscribe(self.on_event1,
HATestClientSession.TEST_TOPIC,
options=SubscribeOptions(match='exact',
details=True))
for i in range(self._loops):
self._sender_loop('{}.{}'.format(self._logname, i))
self.log.info(
'{logname} ready [period={period}, loops={loops}, rate={rate}, stride={stride}, size={size}]',
logname=self._logname,
period=self._period,
loops=self._loops,
rate=self._rate,
stride=self._stride,
size=self._size)
class NativeProcessSession(ApplicationSession):
"""
A native Crossbar.io process (currently: controller, router or container).
"""
def onConnect(self, do_join = True):
"""
"""
if not hasattr(self, 'debug'):
self.debug = self.config.extra.debug
if not hasattr(self, 'cbdir'):
self.cbdir = self.config.extra.cbdir
if not hasattr(self, '_uri_prefix'):
self._uri_prefix = 'crossbar.node.{}'.format(self.config.extra.node)
if self.debug:
log.msg("Session connected to management router")
self._started = datetime.utcnow()
## see: BaseSession
self.include_traceback = False
self.debug_app = False
self._manhole_service = None
if _HAS_PSUTIL:
self._pinfo = ProcessInfo()
self._pinfo_monitor = None
self._pinfo_monitor_seq = 0
else:
self._pinfo = None
self._pinfo_monitor = None
self._pinfo_monitor_seq = None
log.msg("Warning: process utilities not available")
if do_join:
self.join(self.config.realm)
@inlineCallbacks
def onJoin(self, details):
"""
Called when process has joined the node's management realm.
"""
procs = [
'start_manhole',
'stop_manhole',
'get_manhole',
'trigger_gc',
'utcnow',
'started',
'uptime',
'get_process_info',
'get_process_stats',
'set_process_stats_monitoring'
]
dl = []
for proc in procs:
uri = '{}.{}'.format(self._uri_prefix, proc)
if self.debug:
log.msg("Registering procedure '{}'".format(uri))
dl.append(self.register(getattr(self, proc), uri, options = RegisterOptions(details_arg = 'details', discloseCaller = True)))
regs = yield DeferredList(dl)
if self.debug:
log.msg("{} registered {} procedures".format(self.__class__.__name__, len(regs)))
def get_process_info(self, details = None):
"""
Get process information (open files, sockets, ...).
:returns: dict -- Dictionary with process information.
"""
if self.debug:
log.msg("{}.get_process_info".format(self.__class__.__name__))
if self._pinfo:
return self._pinfo.get_info()
else:
emsg = "ERROR: could not retrieve process statistics - required packages not installed"
raise ApplicationError("crossbar.error.feature_unavailable", emsg)
def get_process_stats(self, details = None):
"""
Get process statistics (CPU, memory, I/O).
:returns: dict -- Dictionary with process statistics.
"""
if self.debug:
log.msg("{}.get_process_stats".format(self.__class__.__name__))
if self._pinfo:
return self._pinfo.get_stats()
else:
emsg = "ERROR: could not retrieve process statistics - required packages not installed"
raise ApplicationError("crossbar.error.feature_unavailable", emsg)
def set_process_stats_monitoring(self, interval, details = None):
"""
Enable/disable periodic publication of process statistics.
:param interval: The monitoring interval in seconds. Set to 0 to disable monitoring.
:type interval: float
"""
if self.debug:
log.msg("{}.set_process_stats_monitoring".format(self.__class__.__name__), interval)
if self._pinfo:
stats_monitor_set_topic = '{}.on_process_stats_monitoring_set'.format(self._uri_prefix)
## stop and remove any existing monitor
if self._pinfo_monitor:
self._pinfo_monitor.stop()
self._pinfo_monitor = None
self.publish(stats_monitor_set_topic, 0, options = PublishOptions(exclude = [details.caller]))
## possibly start a new monitor
if interval > 0:
stats_topic = '{}.on_process_stats'.format(self._uri_prefix)
def publish_stats():
stats = self._pinfo.get_stats()
self._pinfo_monitor_seq += 1
stats['seq'] = self._pinfo_monitor_seq
self.publish(stats_topic, stats)
self._pinfo_monitor = LoopingCall(publish_stats)
self._pinfo_monitor.start(interval)
self.publish(stats_monitor_set_topic, interval, options = PublishOptions(exclude = [details.caller]))
else:
emsg = "ERROR: cannot setup process statistics monitor - required packages not installed"
raise ApplicationError("crossbar.error.feature_unavailable", emsg)
def trigger_gc(self, details = None):
"""
Triggers a garbage collection.
:returns: float -- Time consumed for GC in ms.
"""
if self.debug:
log.msg("{}.trigger_gc".format(self.__class__.__name__))
started = rtime()
gc.collect()
return 1000. * (rtime() - started)
@inlineCallbacks
def start_manhole(self, config, details = None):
"""
Start a manhole (SSH) within this worker.
:param config: Manhole configuration.
:type config: obj
"""
if self.debug:
log.msg("{}.start_manhole".format(self.__class__.__name__), config)
if not _HAS_MANHOLE:
emsg = "ERROR: could not start manhole - required packages are missing ({})".format(_MANHOLE_MISSING_REASON)
log.msg(emsg)
raise ApplicationError("crossbar.error.feature_unavailable", emsg)
if self._manhole_service:
emsg = "ERROR: could not start manhole - already running (or starting)"
log.msg(emsg)
raise ApplicationError("crossbar.error.already_started", emsg)
try:
checkconfig.check_manhole(config)
except Exception as e:
emsg = "ERROR: could not start manhole - invalid configuration ({})".format(e)
log.msg(emsg)
raise ApplicationError('crossbar.error.invalid_configuration', emsg)
## setup user authentication
##
checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()
for user in config['users']:
checker.addUser(user['user'], user['password'])
## setup manhole namespace
##
namespace = {'session': self}
class PatchedTerminalSession(TerminalSession):
## get rid of
## exceptions.AttributeError: TerminalSession instance has no attribute 'windowChanged'
def windowChanged(self, winSize):
pass
rlm = TerminalRealm()
rlm.sessionFactory = PatchedTerminalSession # monkey patch
rlm.chainedProtocolFactory.protocolFactory = lambda _: ColoredManhole(namespace)
ptl = portal.Portal(rlm, [checker])
factory = ConchFactory(ptl)
factory.noisy = False
self._manhole_service = ManholeService(config, details.authid)
starting_topic = '{}.on_manhole_starting'.format(self._uri_prefix)
starting_info = self._manhole_service.marshal()
## the caller gets a progressive result ..
if details.progress:
details.progress(starting_info)
## .. while all others get an event
self.publish(starting_topic, starting_info, options = PublishOptions(exclude = [details.caller]))
try:
self._manhole_service.port = yield create_listening_port_from_config(config['endpoint'], factory, self.cbdir, reactor)
except Exception as e:
self._manhole_service = None
emsg = "ERROR: manhole service endpoint cannot listen - {}".format(e)
log.msg(emsg)
raise ApplicationError("crossbar.error.cannot_listen", emsg)
## alright, manhole has started
self._manhole_service.started = datetime.utcnow()
self._manhole_service.status = 'started'
started_topic = '{}.on_manhole_started'.format(self._uri_prefix)
started_info = self._manhole_service.marshal()
self.publish(started_topic, started_info, options = PublishOptions(exclude = [details.caller]))
returnValue(started_info)
@inlineCallbacks
def stop_manhole(self, details = None):
"""
Stop Manhole.
"""
if self.debug:
log.msg("{}.stop_manhole".format(self.__class__.__name__))
if not _HAS_MANHOLE:
emsg = "ERROR: could not start manhole - required packages are missing ({})".format(_MANHOLE_MISSING_REASON)
log.msg(emsg)
raise ApplicationError("crossbar.error.feature_unavailable", emsg)
if not self._manhole_service or self._manhole_service.status != 'started':
emsg = "ERROR: cannot stop manhole - not running (or already shutting down)"
raise ApplicationError("crossbar.error.not_started", emsg)
self._manhole_service.status = 'stopping'
stopping_topic = '{}.on_manhole_stopping'.format(self._uri_prefix)
stopping_info = None
## the caller gets a progressive result ..
if details.progress:
details.progress(stopping_info)
## .. while all others get an event
self.publish(stopping_topic, stopping_info, options = PublishOptions(exclude = [details.caller]))
try:
yield self._manhole_service.port.stopListening()
except Exception as e:
raise Exception("INTERNAL ERROR: don't know how to handle a failed called to stopListening() - {}".format(e))
self._manhole_service = None
stopped_topic = '{}.on_manhole_stopped'.format(self._uri_prefix)
stopped_info = None
self.publish(stopped_topic, stopped_info, options = PublishOptions(exclude = [details.caller]))
returnValue(stopped_info)
def get_manhole(self, details = None):
"""
Get current manhole service information.
:returns: dict -- A dict with service information or `None` if the service is not running.
"""
if self.debug:
log.msg("{}.get_manhole".format(self.__class__.__name__))
if not _HAS_MANHOLE:
emsg = "ERROR: could not start manhole - required packages are missing ({})".format(_MANHOLE_MISSING_REASON)
log.msg(emsg)
raise ApplicationError("crossbar.error.feature_unavailable", emsg)
if not self._manhole_service:
return None
else:
return self._manhole_service.marshal()
def utcnow(self, details = None):
"""
Return current time as determined from within this process.
:returns str -- Current time (UTC) in UTC ISO 8601 format.
"""
if self.debug:
log.msg("{}.utcnow".format(self.__class__.__name__))
return utcnow()
def started(self, details = None):
"""
Return start time of this process.
:returns str -- Start time (UTC) in UTC ISO 8601 format.
"""
if self.debug:
log.msg("{}.started".format(self.__class__.__name__))
return utcstr(self._started)
def uptime(self, details = None):
"""
Uptime of this process.
:returns float -- Uptime in seconds.
"""
if self.debug:
log.msg("{}.uptime".format(self.__class__.__name__))
now = datetime.utcnow()
return (now - self._started).total_seconds()
class NativeProcessSession(ApplicationSession):
"""
A native Crossbar.io process (currently: controller, router or container).
"""
log = make_logger()
def onConnect(self, do_join=True):
"""
"""
if not hasattr(self, 'cbdir'):
self.cbdir = self.config.extra.cbdir
if not hasattr(self, '_uri_prefix'):
self._uri_prefix = 'crossbar.node.{}'.format(self.config.extra.node)
self._started = datetime.utcnow()
# see: BaseSession
self.include_traceback = False
self.debug_app = False
self._manhole_service = None
if _HAS_PSUTIL:
self._pinfo = ProcessInfo()
self._pinfo_monitor = None
self._pinfo_monitor_seq = 0
else:
self._pinfo = None
self._pinfo_monitor = None
self._pinfo_monitor_seq = None
self.log.info("Process utilities not available")
self._connections = {}
if do_join:
self.join(self.config.realm)
@inlineCallbacks
def onJoin(self, details):
"""
Called when process has joined the node's management realm.
"""
procs = [
'start_manhole',
'stop_manhole',
'get_manhole',
'start_connection',
'stop_connection',
'get_connections',
'trigger_gc',
'utcnow',
'started',
'uptime',
'get_process_info',
'get_process_stats',
'set_process_stats_monitoring'
]
dl = []
for proc in procs:
uri = '{}.{}'.format(self._uri_prefix, proc)
self.log.debug("Registering procedure '{uri}'", uri=uri)
dl.append(self.register(getattr(self, proc), uri, options=RegisterOptions(details_arg='details')))
regs = yield DeferredList(dl)
self.log.debug("Registered {len_reg} procedures", len_reg=len(regs))
@inlineCallbacks
def start_connection(self, id, config, details=None):
"""
Starts a connection in this process.
:param id: The ID for the started connection.
:type id: unicode
:param config: Connection configuration.
:type config: dict
:param details: Caller details.
:type details: instance of :class:`autobahn.wamp.types.CallDetails`
:returns dict -- The connection.
"""
self.log.debug("start_connection: id={id}, config={config}", id=id, config=config)
# prohibit starting a component twice
#
if id in self._connections:
emsg = "cannot start connection: a connection with id={} is already started".format(id)
self.log.warn(emsg)
raise ApplicationError("crossbar.error.invalid_configuration", emsg)
# check configuration
#
try:
checkconfig.check_connection(config)
except Exception as e:
emsg = "invalid connection configuration ({})".format(e)
self.log.warn(emsg)
raise ApplicationError("crossbar.error.invalid_configuration", emsg)
else:
self.log.info("Starting {} in process.".format(config['type']))
if config['type'] == u'postgresql.connection':
if _HAS_POSTGRESQL:
connection = PostgreSQLConnection(id, config)
else:
emsg = "unable to start connection - required PostgreSQL driver package not installed"
self.log.warn(emsg)
raise ApplicationError("crossbar.error.feature_unavailable", emsg)
else:
# should not arrive here
raise Exception("logic error")
self._connections[id] = connection
try:
yield connection.start()
self.log.info("Connection {connection_type} started '{connection_id}'", connection_id=id, connection_type=config['type'])
except Exception as e:
del self._connections[id]
raise
state = connection.marshal()
self.publish(u'crossbar.node.process.on_connection_start', state)
returnValue(state)
@inlineCallbacks
def stop_connection(self, id, details=None):
"""
Stop a connection currently running within this process.
:param id: The ID of the connection to stop.
:type id: unicode
:param details: Caller details.
:type details: instance of :class:`autobahn.wamp.types.CallDetails`
:returns dict -- A dict with component start information.
"""
self.log.debug("stop_connection: id={id}", id=id)
if id not in self._connections:
raise ApplicationError('crossbar.error.no_such_object', 'no connection with ID {} running in this process'.format(id))
connection = self._connections[id]
try:
yield connection.stop()
except Exception as e:
self.log.warn('could not stop connection {id}: {error}', error=e)
raise
del self._connections[id]
state = connection.marshal()
self.publish(u'crossbar.node.process.on_connection_stop', state)
returnValue(state)
def get_connections(self, details=None):
"""
Get connections currently running within this processs.
:param details: Caller details.
:type details: instance of :class:`autobahn.wamp.types.CallDetails`
:returns list -- List of connections.
"""
self.log.debug("get_connections")
res = []
for c in self._connections.values():
res.append(c.marshal())
return res
def get_process_info(self, details=None):
"""
Get process information (open files, sockets, ...).
:returns: dict -- Dictionary with process information.
"""
self.log.debug("{cls}.get_process_info",
cls=self.__class__.__name__)
if self._pinfo:
return self._pinfo.get_info()
else:
emsg = "ERROR: could not retrieve process statistics - required packages not installed"
raise ApplicationError("crossbar.error.feature_unavailable", emsg)
def get_process_stats(self, details=None):
"""
Get process statistics (CPU, memory, I/O).
:returns: dict -- Dictionary with process statistics.
"""
self.log.debug("{cls}.get_process_stats", cls=self.__class__.__name__)
if self._pinfo:
return self._pinfo.get_stats()
else:
emsg = "ERROR: could not retrieve process statistics - required packages not installed"
raise ApplicationError("crossbar.error.feature_unavailable", emsg)
def set_process_stats_monitoring(self, interval, details=None):
"""
Enable/disable periodic publication of process statistics.
:param interval: The monitoring interval in seconds. Set to 0 to disable monitoring.
:type interval: float
"""
self.log.debug("{cls}.set_process_stats_monitoring(interval = {interval})",
cls=self.__class__.__name__, interval=interval)
if self._pinfo:
stats_monitor_set_topic = '{}.on_process_stats_monitoring_set'.format(self._uri_prefix)
# stop and remove any existing monitor
if self._pinfo_monitor:
self._pinfo_monitor.stop()
self._pinfo_monitor = None
self.publish(stats_monitor_set_topic, 0, options=PublishOptions(exclude=[details.caller]))
# possibly start a new monitor
if interval > 0:
stats_topic = '{}.on_process_stats'.format(self._uri_prefix)
def publish_stats():
stats = self._pinfo.get_stats()
self._pinfo_monitor_seq += 1
stats['seq'] = self._pinfo_monitor_seq
self.publish(stats_topic, stats)
self._pinfo_monitor = LoopingCall(publish_stats)
self._pinfo_monitor.start(interval)
self.publish(stats_monitor_set_topic, interval, options=PublishOptions(exclude=[details.caller]))
else:
emsg = "ERROR: cannot setup process statistics monitor - required packages not installed"
raise ApplicationError("crossbar.error.feature_unavailable", emsg)
def trigger_gc(self, details=None):
"""
Triggers a garbage collection.
:returns: float -- Time consumed for GC in ms.
"""
self.msg.debug("{cls}.trigger_gc", cls=self.__class__.__name__)
started = rtime()
gc.collect()
return 1000. * (rtime() - started)
@inlineCallbacks
def start_manhole(self, config, details=None):
"""
Start a manhole (SSH) within this worker.
:param config: Manhole configuration.
:type config: obj
"""
self.log.debug("{cls}.start_manhole(config = {config})",
cls=self.__class__.__name__, config=config)
if not _HAS_MANHOLE:
emsg = "ERROR: could not start manhole - required packages are missing ({})".format(_MANHOLE_MISSING_REASON)
self.log.error(emsg)
raise ApplicationError("crossbar.error.feature_unavailable", emsg)
if self._manhole_service:
emsg = "ERROR: could not start manhole - already running (or starting)"
self.log.warn(emsg)
raise ApplicationError("crossbar.error.already_started", emsg)
try:
checkconfig.check_manhole(config)
except Exception as e:
emsg = "ERROR: could not start manhole - invalid configuration ({})".format(e)
self.log.error(emsg)
raise ApplicationError('crossbar.error.invalid_configuration', emsg)
# setup user authentication
#
checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()
for user in config['users']:
checker.addUser(user['user'], user['password'])
# setup manhole namespace
#
namespace = {'session': self}
class PatchedTerminalSession(TerminalSession):
# get rid of
# exceptions.AttributeError: TerminalSession instance has no attribute 'windowChanged'
def windowChanged(self, winSize):
pass
rlm = TerminalRealm()
rlm.sessionFactory = PatchedTerminalSession # monkey patch
rlm.chainedProtocolFactory.protocolFactory = lambda _: ColoredManhole(namespace)
ptl = portal.Portal(rlm, [checker])
factory = ConchFactory(ptl)
factory.noisy = False
self._manhole_service = ManholeService(config, details.caller)
starting_topic = '{}.on_manhole_starting'.format(self._uri_prefix)
starting_info = self._manhole_service.marshal()
# the caller gets a progressive result ..
if details.progress:
details.progress(starting_info)
# .. while all others get an event
self.publish(starting_topic, starting_info, options=PublishOptions(exclude=[details.caller]))
try:
self._manhole_service.port = yield create_listening_port_from_config(config['endpoint'], factory, self.cbdir, reactor)
except Exception as e:
self._manhole_service = None
emsg = "ERROR: manhole service endpoint cannot listen - {}".format(e)
self.log.error(emsg)
raise ApplicationError("crossbar.error.cannot_listen", emsg)
# alright, manhole has started
self._manhole_service.started = datetime.utcnow()
self._manhole_service.status = 'started'
started_topic = '{}.on_manhole_started'.format(self._uri_prefix)
started_info = self._manhole_service.marshal()
self.publish(started_topic, started_info, options=PublishOptions(exclude=[details.caller]))
returnValue(started_info)
@inlineCallbacks
def stop_manhole(self, details=None):
"""
Stop Manhole.
"""
self.log.debug("{cls}.stop_manhole", cls=self.__class__.__name__)
if not _HAS_MANHOLE:
emsg = "ERROR: could not start manhole - required packages are missing ({})".format(_MANHOLE_MISSING_REASON)
self.log.error(emsg)
raise ApplicationError("crossbar.error.feature_unavailable", emsg)
if not self._manhole_service or self._manhole_service.status != 'started':
emsg = "ERROR: cannot stop manhole - not running (or already shutting down)"
raise ApplicationError("crossbar.error.not_started", emsg)
self._manhole_service.status = 'stopping'
stopping_topic = '{}.on_manhole_stopping'.format(self._uri_prefix)
stopping_info = None
# the caller gets a progressive result ..
if details.progress:
details.progress(stopping_info)
# .. while all others get an event
self.publish(stopping_topic, stopping_info, options=PublishOptions(exclude=[details.caller]))
try:
yield self._manhole_service.port.stopListening()
except Exception as e:
raise Exception("INTERNAL ERROR: don't know how to handle a failed called to stopListening() - {}".format(e))
self._manhole_service = None
stopped_topic = '{}.on_manhole_stopped'.format(self._uri_prefix)
stopped_info = None
self.publish(stopped_topic, stopped_info, options=PublishOptions(exclude=[details.caller]))
returnValue(stopped_info)
def get_manhole(self, details=None):
"""
Get current manhole service information.
:returns: dict -- A dict with service information or `None` if the service is not running.
"""
self.log.debug("{cls}.get_manhole", cls=self.__class__.__name__)
if not _HAS_MANHOLE:
emsg = "ERROR: could not start manhole - required packages are missing ({})".format(_MANHOLE_MISSING_REASON)
self.log.error(emsg)
raise ApplicationError("crossbar.error.feature_unavailable", emsg)
if not self._manhole_service:
return None
else:
return self._manhole_service.marshal()
def utcnow(self, details=None):
"""
Return current time as determined from within this process.
:returns str -- Current time (UTC) in UTC ISO 8601 format.
"""
self.log.debug("{cls}.utcnow", cls=self.__class__.__name__)
return utcnow()
def started(self, details=None):
"""
Return start time of this process.
:returns str -- Start time (UTC) in UTC ISO 8601 format.
"""
self.log.debug("{cls}.started", cls=self.__class__.__name__)
return utcstr(self._started)
def uptime(self, details=None):
"""
Uptime of this process.
:returns float -- Uptime in seconds.
"""
self.log.debug("{cls}.uptime", cls=self.__class__.__name__)
now = datetime.utcnow()
return (now - self._started).total_seconds()
class AppSession(ApplicationSession):
@inlineCallbacks
def onJoin(self, details):
self._db = Database(dbpath='../results')
self._schema = Schema.attach(self._db)
self._pinfo = ProcessInfo()
self._logname = self.config.extra['logname']
self._period = self.config.extra.get('period', 5.)
self._running = True
dl = []
for i in range(8):
d = self._loop(i)
dl.append(d)
d = gatherResults(dl)
try:
yield d
except TransportLost:
pass
def onLeave(self, details):
self._running = False
@inlineCallbacks
def _loop(self, index):
prefix = self.config.extra['prefix']
last = None
while self._running:
rtts = []
batch_started_str = utcnow()
batch_started = time()
while (time() - batch_started) < self._period:
ts_req = time_ns()
res = yield self.call('{}.echo'.format(prefix), ts_req)
ts_res = time_ns()
assert res == ts_req
rtt = ts_res - ts_req
rtts.append(rtt)
stats = self._pinfo.get_stats()
if last:
batch_duration = (stats['time'] - last['time']) / 10 ** 9
ctx = round((stats['voluntary'] - last['voluntary']) / batch_duration, 0)
self.log.info('{logname}: {cpu} cpu, {mem} mem, {ctx} ctx',
logname=self._logname,
cpu=round(stats['cpu_percent'], 1),
mem=round(stats['mem_percent'], 1),
ctx=ctx)
rtts = sorted(rtts)
sr = WampStatsRecord()
sr.key = '{}#{}.{}'.format(batch_started_str, self._logname, index)
sr.count = len(rtts)
sr.calls_per_sec = int(round(sr.count / batch_duration, 0))
# all times here are in microseconds:
sr.avg_rtt = round(1000000. * batch_duration / float(sr.count), 1)
sr.max_rtt = round(rtts[-1] / 1000, 1)
sr.q50_rtt = round(rtts[int(sr.count / 2.)] / 1000, 1)
sr.q99_rtt = round(rtts[int(-(sr.count / 100.))] / 1000, 1)
sr.q995_rtt = round(rtts[int(-(sr.count / 995.))] / 1000, 1)
with self._db.begin(write=True) as txn:
self._schema.wamp_stats[txn, sr.key] = sr
print("{}: {} calls, {} calls/sec, RTT (us): q50 {}, avg {}, q99 {}, q995 {}, max {}".format(sr.key,
sr.count,
sr.calls_per_sec,
sr.q50_rtt,
sr.avg_rtt,
sr.q99_rtt,
sr.q995_rtt,
sr.max_rtt))
last = stats
