def __init__(self, factory, config, templates):
"""
Ctor.
:param factory: WAMP session factory.
:type factory: An instance of ..
:param config: Crossbar transport configuration.
:type config: dict
"""
options = config.get('options', {})
server = "Crossbar/{}".format(crossbar.__version__)
externalPort = options.get('external_port', None)
WampWebSocketServerFactory.__init__(self,
factory,
url = config.get('url', None),
server = server,
externalPort = externalPort,
debug = config.get('debug', False))
## transport configuration
self._config = config
## Jinja2 templates for 404 etc
self._templates = templates
## cookie tracking
if 'cookie' in config:
self._cookies = {}
## set WebSocket options
set_websocket_options(self, options)
def __init__(self, factory, *args, **kwargs):
self._connection_count = 0
self._timeout = kwargs['timeout']
self._reaper = reactor.callLater(self._timeout, lambda: reactor.stop())
del kwargs['timeout']
WampWebSocketServerFactory.__init__(self, factory, *args, **kwargs)
WampWebSocketServerFactory.protocol = TimeoutWampWebSocketServerProtocol
def __init__(self, factory, *args, **kwargs):
self._connection_count = 0
self._timeout = kwargs['timeout']
self._reaper = reactor.callLater(self._timeout, lambda: reactor.stop())
del kwargs['timeout']
WampWebSocketServerFactory.__init__(self, factory, *args, **kwargs)
WampWebSocketServerFactory.protocol = TimeoutWampWebSocketServerProtocol
def run(self, make, start_reactor = True):
"""
Run the application component.
:param make: A factory that produces instances of :class:`autobahn.asyncio.wamp.ApplicationSession`
when called with an instance of :class:`autobahn.wamp.types.ComponentConfig`.
:type make: callable
"""
from twisted.internet import reactor
isSecure, host, port, resource, path, params = parseWsUrl(self.url)
## start logging to console
if self.debug or self.debug_wamp or self.debug_app:
log.startLogging(sys.stdout)
## run an embedded router if ask to start standalone
if self.standalone:
from autobahn.wamp.router import RouterFactory
from autobahn.twisted.websocket import WampWebSocketServerFactory
from twisted.internet.endpoints import serverFromString
router_factory = RouterFactory()
session_factory = RouterSessionFactory(router_factory)
transport_factory = WampWebSocketServerFactory(session_factory, debug = self.debug, debug_wamp = self.debug_wamp)
transport_factory.setProtocolOptions(failByDrop = False)
server = serverFromString(reactor, "tcp:{}".format(port))
server.listen(transport_factory)
## factory for use ApplicationSession
def create():
cfg = ComponentConfig(self.realm, self.extra)
try:
session = make(cfg)
except Exception:
## the app component could not be created .. fatal
log.err()
reactor.stop()
else:
session.debug_app = self.debug_app
return session
## create a WAMP-over-WebSocket transport client factory
transport_factory = WampWebSocketClientFactory(create, url = self.url,
debug = self.debug, debug_wamp = self.debug_wamp)
## start the client from a Twisted endpoint
from twisted.internet.endpoints import clientFromString
client = clientFromString(reactor, "tcp:{}:{}".format(host, port))
client.connect(transport_factory)
## now enter the Twisted reactor loop
if start_reactor:
reactor.run()
def start_management_transport(self, config, details = None):
"""
Start transport for local management router.
:param config: Transport configuration.
:type config: obj
"""
if self.debug:
log.msg("{}.start_management_transport".format(self.__class__.__name__), config)
if self._management_transport:
emsg = "ERROR: could not start management transport - already running (or starting)"
log.msg(emsg)
raise ApplicationError("crossbar.error.already_started", emsg)
try:
checkconfig.check_listening_transport_websocket(config)
except Exception as e:
emsg = "ERROR: could not start management transport - invalid configuration ({})".format(e)
log.msg(emsg)
raise ApplicationError('crossbar.error.invalid_configuration', emsg)
self._management_transport = ManagementTransport(config, details.authid)
factory = WampWebSocketServerFactory(self._node._router_session_factory, debug = False)
factory.setProtocolOptions(failByDrop = False)
factory.noisy = False
starting_topic = '{}.on_management_transport_starting'.format(self._uri_prefix)
starting_info = self._management_transport.marshal()
## the caller gets a progressive result ..
if details.progress:
details.progress(starting_info)
## .. while all others get an event
self.publish(starting_topic, starting_info, options = PublishOptions(exclude = [details.caller]))
try:
self._management_transport.port = yield create_listening_port_from_config(config['endpoint'], factory, self.cbdir, reactor)
except Exception as e:
self._management_transport = None
emsg = "ERROR: local management service endpoint cannot listen - {}".format(e)
log.msg(emsg)
raise ApplicationError("crossbar.error.cannot_listen", emsg)
## alright, manhole has started
self._management_transport.started = datetime.utcnow()
self._management_transport.status = 'started'
started_topic = '{}.on_management_transport_started'.format(self._uri_prefix)
started_info = self._management_transport.marshal()
self.publish(started_topic, started_info, options = PublishOptions(exclude = [details.caller]))
returnValue(started_info)
def start_management_transport(self, config, details=None):
"""
Start a (listening) transport for the local management router.
:param config: Transport configuration.
:type config: obj
"""
if self.debug:
self.log.debug("{me}.start_management_transport",
me=self.__class__.__name__, config=config)
if self._management_transport:
emsg = "ERROR: could not start management transport - already running (or starting)"
self.log.failure(emsg)
raise ApplicationError("crossbar.error.already_started", emsg)
try:
checkconfig.check_listening_transport_websocket(config)
except Exception as e:
emsg = "ERROR: could not start management transport - invalid configuration ({})".format(e)
self.log.failure(emsg)
raise ApplicationError('crossbar.error.invalid_configuration', emsg)
self._management_transport = ManagementTransport(config, details.caller)
factory = WampWebSocketServerFactory(self._node._router_session_factory, debug=False)
factory.setProtocolOptions(failByDrop=False)
factory.noisy = False
starting_topic = '{}.on_management_transport_starting'.format(self._uri_prefix)
starting_info = self._management_transport.marshal()
# the caller gets a progressive result ..
if details.progress:
details.progress(starting_info)
# .. while all others get an event
self.publish(starting_topic, starting_info, options=PublishOptions(exclude=[details.caller]))
try:
self._management_transport.port = yield create_listening_port_from_config(config['endpoint'], factory, self.cbdir, reactor)
except Exception as e:
self._management_transport = None
emsg = "ERROR: local management service endpoint cannot listen - {}".format(e)
self.log.failure(emsg)
raise ApplicationError("crossbar.error.cannot_listen", emsg)
# alright, the transport has started
self._management_transport.started = datetime.utcnow()
self._management_transport.status = 'started'
started_topic = '{}.on_management_transport_started'.format(self._uri_prefix)
started_info = self._management_transport.marshal()
self.publish(started_topic, started_info, options=PublishOptions(exclude=[details.caller]))
returnValue(started_info)
def create(transport, path, config, controller):
# this is the crossbar-specific wamp-websocket-server
# from crossbar.router.protocol import WampWebSocketServerFactory
from autobahn.twisted.websocket import WampWebSocketServerFactory
websocket_factory = WampWebSocketServerFactory(FrontendProxySession)
websocket_factory.protocol = Frontend
websocket_factory._controller = controller
resource = WebSocketResource(websocket_factory)
return ProxyWebSocketService(transport, path, config, resource)
class WampServer:
def __init__(self, debug = '0'):
self.debug = debug
log.startLogging(sys.stdout)
## we use an Autobahn utility to install the "best" available Twisted reactor
##
from autobahn.twisted.choosereactor import install_reactor
self.reactor = install_reactor()
print("Running on reactor {}".format(self.reactor))
## create a WAMP router factory
##
from autobahn.twisted.wamp import RouterFactory
self.router_factory = RouterFactory()
## create a WAMP router session factory
##
from autobahn.twisted.wamp import RouterSessionFactory
self.session_factory = RouterSessionFactory(self.router_factory)
## Add embedded WAMP application sessions to the router
##
from core.rpc import MyBackendComponent
component_config = types.ComponentConfig(realm = "realm1")
component_session = MyBackendComponent(component_config)
self.session_factory.add(component_session)
## create a WAMP-over-WebSocket transport server factory
##
from autobahn.twisted.websocket import WampWebSocketServerFactory
self.transport_factory = WampWebSocketServerFactory(self.session_factory, debug = self.debug)
self.transport_factory.setProtocolOptions(failByDrop = False)
def run(self, config):
## start the server from an endpoint
##
enpoint = 'tcp:' + config.PORT
print enpoint
self.server = serverFromString(self.reactor, enpoint)
self.server.listen(self.transport_factory)
## now enter the Twisted reactor loop
##
self.reactor.run()
def start(self):
"""
Starts this node. This will start a node controller
and then spawn new worker processes as needed.
The node controller will watch spawned processes,
communicate via stdio with the worker, and start
and restart the worker processes as needed.
"""
## the node controller singleton WAMP application session
##
self._node_controller_session = NodeControllerSession(self)
## router and factory that creates router sessions
##
self._router_factory = RouterFactory()
self._router_session_factory = RouterSessionFactory(self._router_factory)
## add the node controller singleton session to the router
##
self._router_session_factory.add(self._node_controller_session)
if True:
## create a WAMP-over-WebSocket transport server factory
##
from autobahn.twisted.websocket import WampWebSocketServerFactory
from twisted.internet.endpoints import serverFromString
self._router_server_transport_factory = WampWebSocketServerFactory(self._router_session_factory, "ws://localhost:9000", debug = False)
self._router_server_transport_factory.setProtocolOptions(failByDrop = False)
## start the WebSocket server from an endpoint
##
self._router_server = serverFromString(self._reactor, "tcp:9000")
self._router_server.listen(self._router_server_transport_factory)
## factory that creates router session transports. these are for clients
## that talk WAMP-WebSocket over pipes with spawned worker processes and
## for any uplink session to a management service
##
self._router_client_transport_factory = WampWebSocketClientFactory(self._router_session_factory, "ws://localhost", debug = False)
self._router_client_transport_factory.setProtocolOptions(failByDrop = False)
if False:
management_session_factory = ApplicationSessionFactory()
management_session_factory.session = NodeManagementSession
management_session_factory.node_controller_session = node_controller_session
management_transport_factory = WampWebSocketClientFactory(management_session_factory, "ws://127.0.0.1:7000")
management_transport_factory.setProtocolOptions(failByDrop = False)
management_client = clientFromString(self._reactor, "tcp:127.0.0.1:7000")
management_client.connect(management_transport_factory)
## startup the node from configuration file
##
self._node_controller_session.run_node_config(self._config)
def wampServ(wampAddress, wampPort, wampDebug = False):
"""
Sets up an Autobahn|Python WAMPv2 server.
Code modified from WAMP documentation.
"""
from twisted.internet.endpoints import serverFromString
from autobahn.wamp.router import RouterFactory
from autobahn.twisted.wamp import RouterSessionFactory
from autobahn.twisted.websocket import WampWebSocketServerFactory
## create a WAMP router factory
router_factory = RouterFactory()
## create a WAMP router session factory
session_factory = RouterSessionFactory(router_factory)
## create a WAMP-over-WebSocket transport server factory
transport_factory = WampWebSocketServerFactory(session_factory, wampAddress, debug = wampDebug)
transport_factory.setProtocolOptions(failByDrop = False)
## Start websocket server
server = serverFromString(reactor, wampPort)
server.listen(transport_factory)
def start_local_management_transport(self, endpoint_descriptor):
## create a WAMP-over-WebSocket transport server factory
##
from autobahn.twisted.websocket import WampWebSocketServerFactory
from twisted.internet.endpoints import serverFromString
self._router_server_transport_factory = WampWebSocketServerFactory(self._router_session_factory, debug = self.debug)
self._router_server_transport_factory.setProtocolOptions(failByDrop = False)
## start the WebSocket server from an endpoint
##
self._router_server = serverFromString(self._reactor, endpoint_descriptor)
self._router_server.listen(self._router_server_transport_factory)
def run():
"""
Entry point into (native) worker processes. This wires up stuff such that
a worker instance is talking WAMP-over-stdio to the node controller.
"""
import os
import sys
import platform
import signal
# Ignore SIGINT so we get consistent behavior on control-C versus
# sending SIGINT to the controller process. When the controller is
# shutting down, it sends TERM to all its children but ctrl-C
# handling will send a SIGINT to all the processes in the group
# (so then the controller sends a TERM but the child already or
# will very shortly get a SIGINT as well). Twisted installs signal
# handlers, but not for SIGINT if there's already a custom one
# present.
def ignore(sig, frame):
log.debug("Ignoring SIGINT in worker.")
signal.signal(signal.SIGINT, ignore)
# create the top-level parser
#
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--reactor',
default=None,
choices=['select', 'poll', 'epoll', 'kqueue', 'iocp'],
help='Explicit Twisted reactor selection (optional).')
parser.add_argument(
'--loglevel',
default="info",
choices=['none', 'error', 'warn', 'info', 'debug', 'trace'],
help='Initial log level.')
parser.add_argument('-c',
'--cbdir',
type=six.text_type,
help="Crossbar.io node directory (required).")
parser.add_argument('-r',
'--realm',
type=six.text_type,
help='Crossbar.io node (management) realm (required).')
parser.add_argument('-t',
'--type',
choices=['router', 'container', 'websocket-testee'],
help='Worker type (required).')
parser.add_argument('-w',
'--worker',
type=six.text_type,
help='Crossbar.io worker ID (required).')
parser.add_argument('--title',
type=six.text_type,
default=None,
help='Worker process title to set (optional).')
parser.add_argument(
'--expose_controller',
type=bool,
default=False,
help=
'Expose node controller session to all components (this feature requires Crossbar.io Fabric extension).'
)
parser.add_argument(
'--expose_shared',
type=bool,
default=False,
help=
'Expose a shared object to all components (this feature requires Crossbar.io Fabric extension).'
)
options = parser.parse_args()
# make sure logging to something else than stdio is setup _first_
#
from crossbar._logging import make_JSON_observer, cb_logging_aware
from txaio import make_logger, start_logging
from twisted.logger import globalLogPublisher
log = make_logger()
# Print a magic phrase that tells the capturing logger that it supports
# Crossbar's rich logging
print(cb_logging_aware, file=sys.__stderr__)
sys.__stderr__.flush()
flo = make_JSON_observer(sys.__stderr__)
globalLogPublisher.addObserver(flo)
start_logging(None, options.loglevel)
# we use an Autobahn utility to import the "best" available Twisted reactor
#
from autobahn.twisted.choosereactor import install_reactor
reactor = install_reactor(options.reactor)
WORKER_TYPE_TO_TITLE = {
'router': 'Router',
'container': 'Container',
'websocket-testee': 'WebSocket Testee'
}
from twisted.python.reflect import qual
log.info(
'{worker_title} worker "{worker_id}" process {pid} starting on {python}-{reactor} ..',
worker_title=WORKER_TYPE_TO_TITLE[options.type],
worker_id=options.worker,
pid=os.getpid(),
python=platform.python_implementation(),
reactor=qual(reactor.__class__).split('.')[-1])
# set process title if requested to
#
try:
import setproctitle
except ImportError:
log.debug(
"Could not set worker process title (setproctitle not installed)")
else:
if options.title:
setproctitle.setproctitle(options.title)
else:
WORKER_TYPE_TO_PROCESS_TITLE = {
'router': 'crossbar-worker [router]',
'container': 'crossbar-worker [container]',
'websocket-testee': 'crossbar-worker [websocket-testee]'
}
setproctitle.setproctitle(
WORKER_TYPE_TO_PROCESS_TITLE[options.type].strip())
# node directory
#
options.cbdir = os.path.abspath(options.cbdir)
os.chdir(options.cbdir)
# log.msg("Starting from node directory {}".format(options.cbdir))
from crossbar.worker.router import RouterWorkerSession
from crossbar.worker.container import ContainerWorkerSession
from crossbar.worker.testee import WebSocketTesteeWorkerSession
WORKER_TYPE_TO_CLASS = {
'router': RouterWorkerSession,
'container': ContainerWorkerSession,
'websocket-testee': WebSocketTesteeWorkerSession
}
from twisted.internet.error import ConnectionDone
from autobahn.twisted.websocket import WampWebSocketServerProtocol
class WorkerServerProtocol(WampWebSocketServerProtocol):
def connectionLost(self, reason):
# the behavior here differs slightly whether we're shutting down orderly
# or shutting down because of "issues"
if isinstance(reason.value, ConnectionDone):
was_clean = True
else:
was_clean = False
try:
# this log message is unlikely to reach the controller (unless
# only stdin/stdout pipes were lost, but not stderr)
if was_clean:
log.info("Connection to node controller closed cleanly")
else:
log.warn("Connection to node controller lost: {reason}",
reason=reason)
# give the WAMP transport a change to do it's thing
WampWebSocketServerProtocol.connectionLost(self, reason)
except:
# we're in the process of shutting down .. so ignore ..
pass
finally:
# after the connection to the node controller is gone,
# the worker is "orphane", and should exit
# determine process exit code
if was_clean:
exit_code = 0
else:
exit_code = 1
# exit the whole worker process when the reactor has stopped
reactor.addSystemEventTrigger('after', 'shutdown', os._exit,
exit_code)
# stop the reactor
try:
reactor.stop()
except ReactorNotRunning:
pass
try:
# create a WAMP application session factory
#
from autobahn.twisted.wamp import ApplicationSessionFactory
from autobahn.wamp.types import ComponentConfig
session_config = ComponentConfig(realm=options.realm, extra=options)
session_factory = ApplicationSessionFactory(session_config)
session_factory.session = WORKER_TYPE_TO_CLASS[options.type]
# create a WAMP-over-WebSocket transport server factory
#
from autobahn.twisted.websocket import WampWebSocketServerFactory
transport_factory = WampWebSocketServerFactory(session_factory,
u'ws://localhost')
transport_factory.protocol = WorkerServerProtocol
transport_factory.setProtocolOptions(failByDrop=False)
# create a protocol instance and wire up to stdio
#
from twisted.python.runtime import platform as _platform
from twisted.internet import stdio
proto = transport_factory.buildProtocol(None)
if _platform.isWindows():
stdio.StandardIO(proto)
else:
stdio.StandardIO(proto, stdout=3)
# now start reactor loop
#
if False:
log.info("vmprof enabled.")
import os
import vmprof
PROFILE_FILE = 'vmprof_{}.dat'.format(os.getpid())
outfd = os.open(PROFILE_FILE, os.O_RDWR | os.O_CREAT | os.O_TRUNC)
vmprof.enable(outfd, period=0.01)
log.info("Entering event loop...")
reactor.run()
vmprof.disable()
else:
log.debug("Entering event loop...")
reactor.run()
except Exception as e:
log.info("Unhandled exception: {e}", e=e)
if reactor.running:
reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1)
reactor.stop()
else:
sys.exit(1)
##
from autobahn.wamp import types
session_factory.add(
SessionKlass(types.ComponentConfig(realm=args.realm)))
if args.transport == "websocket":
## create a WAMP-over-WebSocket transport server factory with longpoll fallback
##
from autobahn.wamp.serializer import JsonSerializer, MsgPackSerializer
from autobahn.twisted.websocket import WampWebSocketServerFactory
from autobahn.twisted.resource import WebSocketResource
from twisted.web.server import Site
from twisted.web.static import File
from autobahn.wamp.http import WampHttpResource
ws_factory = WampWebSocketServerFactory(session_factory,
debug_wamp=args.debug)
ws_factory.setProtocolOptions(failByDrop=False)
serializers = [MsgPackSerializer(), JsonSerializer()]
resource = WampHttpResource(serializers,
debug=True,
timeout=100,
killAfter=120)
resource.factory = ws_factory
root = File("longpoll")
root.putChild("ws", WebSocketResource(ws_factory))
root.putChild("longpoll", resource)
if args.static_dir:
#root.putChild("web", File("/Users/arno/Documents/PycharmProjects/AutobahnJS/test/"))
#root.putChild("autobahn.js", File("/Users/arno/Documents/PycharmProjects/AutobahnJS/build/autobahn.js"))
def _run_command_exec_worker(options, reactor=None, personality=None):
"""
Entry point into (native) worker processes. This wires up stuff such that
a worker instance is talking WAMP-over-stdio to the node controller.
"""
import os
import sys
import platform
import signal
# https://coverage.readthedocs.io/en/coverage-4.4.2/subprocess.html#measuring-sub-processes
MEASURING_COVERAGE = False
if 'COVERAGE_PROCESS_START' in os.environ:
try:
import coverage
except ImportError:
pass
else:
# The following will read the environment variable COVERAGE_PROCESS_START,
# and that should be set to the .coveragerc file:
#
# export COVERAGE_PROCESS_START=${PWD}/.coveragerc
#
coverage.process_startup()
MEASURING_COVERAGE = True
# we use an Autobahn utility to import the "best" available Twisted reactor
from autobahn.twisted.choosereactor import install_reactor
reactor = install_reactor(options.reactor)
# make sure logging to something else than stdio is setup _first_
from crossbar._logging import make_JSON_observer, cb_logging_aware
from txaio import make_logger, start_logging
from twisted.logger import globalLogPublisher
from twisted.python.reflect import qual
log = make_logger()
# Print a magic phrase that tells the capturing logger that it supports
# Crossbar's rich logging
print(cb_logging_aware, file=sys.__stderr__)
sys.__stderr__.flush()
flo = make_JSON_observer(sys.__stderr__)
globalLogPublisher.addObserver(flo)
# Ignore SIGINT so we get consistent behavior on control-C versus
# sending SIGINT to the controller process. When the controller is
# shutting down, it sends TERM to all its children but ctrl-C
# handling will send a SIGINT to all the processes in the group
# (so then the controller sends a TERM but the child already or
# will very shortly get a SIGINT as well). Twisted installs signal
# handlers, but not for SIGINT if there's already a custom one
# present.
def ignore(sig, frame):
log.debug("Ignoring SIGINT in worker.")
signal.signal(signal.SIGINT, ignore)
# actually begin logging
start_logging(None, options.loglevel)
# now check if we can import the requested personality
Personality = crossbar.personalities().get(options.personality, None)
if not Personality:
raise Exception('logic error: personality "{}" not installed'.format(options.personality))
# eg: crossbar.worker.container.ContainerController
l = options.klass.split('.')
worker_module, worker_klass = '.'.join(l[:-1]), l[-1]
# now load the worker module and class
_mod = importlib.import_module(worker_module)
klass = getattr(_mod, worker_klass)
log.info(
'Starting worker "{worker_id}" for node "{node_id}" with personality "{personality}" {worker_class}',
worker_id=options.worker,
node_id=options.node,
personality=Personality.NAME,
worker_class=hltype(klass),
)
log.info(
'Running as PID {pid} on {python}-{reactor}',
pid=os.getpid(),
python=platform.python_implementation(),
reactor=qual(reactor.__class__).split('.')[-1],
)
if MEASURING_COVERAGE:
log.info(hl('Code coverage measurements enabled (coverage={coverage_version}).', color='green', bold=True),
coverage_version=coverage.__version__)
# set process title if requested to
#
try:
import setproctitle
except ImportError:
log.debug("Could not set worker process title (setproctitle not installed)")
else:
if options.title:
setproctitle.setproctitle(options.title)
else:
setproctitle.setproctitle('crossbar-worker [{}]'.format(options.klass))
# node directory
#
options.cbdir = os.path.abspath(options.cbdir)
os.chdir(options.cbdir)
# log.msg("Starting from node directory {}".format(options.cbdir))
# set process title if requested to
#
try:
import setproctitle
except ImportError:
log.debug("Could not set worker process title (setproctitle not installed)")
else:
if options.title:
setproctitle.setproctitle(options.title)
else:
setproctitle.setproctitle(
'crossbar-worker [{}]'.format(options.klass)
)
from twisted.internet.error import ConnectionDone
from autobahn.twisted.websocket import WampWebSocketServerProtocol
class WorkerServerProtocol(WampWebSocketServerProtocol):
def connectionLost(self, reason):
# the behavior here differs slightly whether we're shutting down orderly
# or shutting down because of "issues"
if isinstance(reason.value, ConnectionDone):
was_clean = True
else:
was_clean = False
try:
# this log message is unlikely to reach the controller (unless
# only stdin/stdout pipes were lost, but not stderr)
if was_clean:
log.info("Connection to node controller closed cleanly")
else:
log.warn("Connection to node controller lost: {reason}", reason=reason)
# give the WAMP transport a change to do it's thing
WampWebSocketServerProtocol.connectionLost(self, reason)
except:
# we're in the process of shutting down .. so ignore ..
pass
finally:
# after the connection to the node controller is gone,
# the worker is "orphane", and should exit
# determine process exit code
if was_clean:
exit_code = 0
else:
exit_code = 1
# exit the whole worker process when the reactor has stopped
reactor.addSystemEventTrigger('after', 'shutdown', os._exit, exit_code)
# stop the reactor
try:
reactor.stop()
except ReactorNotRunning:
pass
try:
# define a WAMP application session factory
#
from autobahn.wamp.types import ComponentConfig
def make_session():
session_config = ComponentConfig(realm=options.realm, extra=options)
session = klass(config=session_config, reactor=reactor, personality=Personality)
return session
# create a WAMP-over-WebSocket transport server factory
#
from autobahn.twisted.websocket import WampWebSocketServerFactory
transport_factory = WampWebSocketServerFactory(make_session, u'ws://localhost')
transport_factory.protocol = WorkerServerProtocol
transport_factory.setProtocolOptions(failByDrop=False)
# create a protocol instance and wire up to stdio
#
from twisted.python.runtime import platform as _platform
from twisted.internet import stdio
proto = transport_factory.buildProtocol(None)
if _platform.isWindows():
stdio.StandardIO(proto)
else:
stdio.StandardIO(proto, stdout=3)
# now start reactor loop
#
if False:
log.info("vmprof enabled.")
import os
import vmprof
PROFILE_FILE = 'vmprof_{}.dat'.format(os.getpid())
outfd = os.open(PROFILE_FILE, os.O_RDWR | os.O_CREAT | os.O_TRUNC)
vmprof.enable(outfd, period=0.01)
log.info(hl('Entering event reactor ...', color='cyan', bold=True))
reactor.run()
vmprof.disable()
else:
log.info(hl('Entering event reactor ...', color='cyan', bold=True))
reactor.run()
except Exception as e:
log.info("Unhandled exception: {e}", e=e)
if reactor.running:
reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1)
reactor.stop()
else:
sys.exit(1)
def test_minimal(self):
embedded_components, client_components = [], [Case2_Backend, Case2_Frontend]
## create a WAMP router factory
##
router_factory = RouterFactory()
## create a WAMP router session factory
##
session_factory = RouterSessionFactory(router_factory)
## .. and create and add an WAMP application session to
## run next to the router
##
config = types.ComponentConfig(realm = self.realm,
extra = {
'caselog': 'case1.log'
}
)
try:
log = io.open('caselog.log', 'w')
except Exception as e:
print(e)
return
# log = io.open(config.extra['caselog'], 'w')
config.log = log
config.dlog = []
config.components = []
config.all_done = []
for C in embedded_components:
one_done = Deferred()
config.all_done.append(one_done)
c = C(config, one_done)
config.components.append(c)
session_factory.add(c)
if self.transport == "websocket":
## create a WAMP-over-WebSocket transport server factory
##
transport_factory = WampWebSocketServerFactory(session_factory, debug_wamp = self.debug)
transport_factory.setProtocolOptions(failByDrop = False, openHandshakeTimeout = 0, closeHandshakeTimeout = 0)
elif self.transport in ['rawsocket-json', 'rawsocket-msgpack']:
## create a WAMP-over-RawSocket transport server factory
##
if self.transport == 'rawsocket-msgpack':
serializer = MsgPackSerializer()
elif self.transport == 'rawsocket-json':
serializer = JsonSerializer()
else:
raise Exception("should not arrive here")
transport_factory = WampRawSocketServerFactory(session_factory, serializer, debug = self.debug)
else:
raise Exception("should not arrive here")
## start the server from an endpoint
##
from twisted.internet import reactor
server = serverFromString(reactor, self.server)
d = server.listen(transport_factory)
def onlisten(port):
config.port = port
d.addCallback(onlisten)
clients = []
clients_d = []
for C in client_components:
## create a WAMP application session factory
##
session_factory = ApplicationSessionFactory(config)
one_done = Deferred()
config.all_done.append(one_done)
def make_make(Klass, done):
def make(config):
c = Klass(config, done)
config.components.append(c)
return c
return make
## .. and set the session class on the factory
##
session_factory.session = make_make(C, one_done)
if self.transport == "websocket":
serializers = [JsonSerializer()]
## create a WAMP-over-WebSocket transport client factory
##
transport_factory = WampWebSocketClientFactory(session_factory, serializers = serializers, url = self.url, debug_wamp = self.debug)
if True:
def maker(Klass):
class TestClientProtocol(WampWebSocketClientProtocol):
def onOpen(self):
self.txcnt = 0
self.rxcnt = 0
WampWebSocketClientProtocol.onOpen(self)
def sendMessage(self, payload, isBinary):
self.txcnt += 1
print("> : {0:>3} : {1:<20} : {3}".format(self.txcnt, Klass.__name__, payload))
WampWebSocketClientProtocol.sendMessage(self, payload, isBinary)
def onMessage(self, payload, isBinary):
self.rxcnt += 1
print("< : {0:>3} : {1:<20} : {2}".format(self.rxcnt, Klass.__name__, payload))
WampWebSocketClientProtocol.onMessage(self, payload, isBinary)
return TestClientProtocol
transport_factory.protocol = maker(C)
else:
transport_factory.protocol = WampWebSocketClientProtocol
transport_factory.setProtocolOptions(failByDrop = False, openHandshakeTimeout = 0, closeHandshakeTimeout = 0)
elif self.transport in ['rawsocket-json', 'rawsocket-msgpack']:
## create a WAMP-over-RawSocket transport client factory
##
if self.transport == 'rawsocket-msgpack':
serializer = MsgPackSerializer()
elif self.transport == 'rawsocket-json':
serializer = JsonSerializer()
else:
raise Exception("should not arrive here")
transport_factory = WampRawSocketClientFactory(session_factory, serializer, debug = self.debug)
## start the client from an endpoint
##
cl = clientFromString(reactor, self.client)
clients_d.append(cl.connect(transport_factory))
clients.append(cl)
config.connected_clients = None
def client_connected(res):
config.connected_clients = [proto for success, proto in res if success]
DeferredList(clients_d).addCallback(client_connected)
d = DeferredList(config.all_done, consumeErrors = True)
#d = config.components[1]._done
def done(_):
log.flush()
log.close()
if config.port:
config.port.stopListening()
if config.connected_clients:
for proto in config.connected_clients:
proto.transport.abortConnection()
print("Log length: {0}".format(len(config.dlog)))
print(config.dlog)
#from twisted.internet import reactor
#reactor.callLater(1, reactor.stop)
def error(err):
print(err)
d.addCallbacks(done, error)
# d2 = Deferred()
return d
def run(options, reactor=None):
"""
Entry point into (native) worker processes. This wires up stuff such that
a worker instance is talking WAMP-over-stdio to the node controller.
"""
import os
import sys
import platform
import signal
# make sure logging to something else than stdio is setup _first_
#
from crossbar._logging import make_JSON_observer, cb_logging_aware
from txaio import make_logger, start_logging
from twisted.logger import globalLogPublisher
from twisted.python.reflect import qual
log = make_logger()
# Print a magic phrase that tells the capturing logger that it supports
# Crossbar's rich logging
print(cb_logging_aware, file=sys.__stderr__)
sys.__stderr__.flush()
flo = make_JSON_observer(sys.__stderr__)
globalLogPublisher.addObserver(flo)
# Ignore SIGINT so we get consistent behavior on control-C versus
# sending SIGINT to the controller process. When the controller is
# shutting down, it sends TERM to all its children but ctrl-C
# handling will send a SIGINT to all the processes in the group
# (so then the controller sends a TERM but the child already or
# will very shortly get a SIGINT as well). Twisted installs signal
# handlers, but not for SIGINT if there's already a custom one
# present.
def ignore(sig, frame):
log.debug("Ignoring SIGINT in worker.")
signal.signal(signal.SIGINT, ignore)
# actually begin logging
start_logging(None, options.loglevel)
# we use an Autobahn utility to import the "best" available Twisted reactor
#
from autobahn.twisted.choosereactor import install_reactor
reactor = install_reactor(options.reactor)
# eg: crossbar.worker.container.ContainerWorkerSession
l = options.klass.split('.')
worker_module, worker_klass = '.'.join(l[:-1]), l[-1]
# now load the worker module and class
_mod = importlib.import_module(worker_module)
klass = getattr(_mod, worker_klass)
log.info(
'Started {worker_title} worker "{worker_id}" on node "{node_id}" [{klass} / {python}-{reactor}]',
worker_title=klass.WORKER_TITLE,
klass=options.klass,
node_id=options.node,
worker_id=options.worker,
pid=os.getpid(),
python=platform.python_implementation(),
reactor=qual(reactor.__class__).split('.')[-1],
)
# set process title if requested to
#
try:
import setproctitle
except ImportError:
log.debug(
"Could not set worker process title (setproctitle not installed)")
else:
if options.title:
setproctitle.setproctitle(options.title)
else:
setproctitle.setproctitle('crossbar-worker [{}]'.format(
options.klass))
# node directory
#
options.cbdir = os.path.abspath(options.cbdir)
os.chdir(options.cbdir)
# log.msg("Starting from node directory {}".format(options.cbdir))
# set process title if requested to
#
try:
import setproctitle
except ImportError:
log.debug(
"Could not set worker process title (setproctitle not installed)")
else:
if options.title:
setproctitle.setproctitle(options.title)
else:
setproctitle.setproctitle('crossbar-worker [{}]'.format(
options.klass))
from twisted.internet.error import ConnectionDone
from autobahn.twisted.websocket import WampWebSocketServerProtocol
class WorkerServerProtocol(WampWebSocketServerProtocol):
def connectionLost(self, reason):
# the behavior here differs slightly whether we're shutting down orderly
# or shutting down because of "issues"
if isinstance(reason.value, ConnectionDone):
was_clean = True
else:
was_clean = False
try:
# this log message is unlikely to reach the controller (unless
# only stdin/stdout pipes were lost, but not stderr)
if was_clean:
log.info("Connection to node controller closed cleanly")
else:
log.warn("Connection to node controller lost: {reason}",
reason=reason)
# give the WAMP transport a change to do it's thing
WampWebSocketServerProtocol.connectionLost(self, reason)
except:
# we're in the process of shutting down .. so ignore ..
pass
finally:
# after the connection to the node controller is gone,
# the worker is "orphane", and should exit
# determine process exit code
if was_clean:
exit_code = 0
else:
exit_code = 1
# exit the whole worker process when the reactor has stopped
reactor.addSystemEventTrigger('after', 'shutdown', os._exit,
exit_code)
# stop the reactor
try:
reactor.stop()
except ReactorNotRunning:
pass
try:
# create a WAMP application session factory
#
from autobahn.twisted.wamp import ApplicationSessionFactory
from autobahn.wamp.types import ComponentConfig
session_config = ComponentConfig(realm=options.realm, extra=options)
session_factory = ApplicationSessionFactory(session_config)
session_factory.session = klass
# create a WAMP-over-WebSocket transport server factory
#
from autobahn.twisted.websocket import WampWebSocketServerFactory
transport_factory = WampWebSocketServerFactory(session_factory,
u'ws://localhost')
transport_factory.protocol = WorkerServerProtocol
transport_factory.setProtocolOptions(failByDrop=False)
# create a protocol instance and wire up to stdio
#
from twisted.python.runtime import platform as _platform
from twisted.internet import stdio
proto = transport_factory.buildProtocol(None)
if _platform.isWindows():
stdio.StandardIO(proto)
else:
stdio.StandardIO(proto, stdout=3)
# now start reactor loop
#
if False:
log.info("vmprof enabled.")
import os
import vmprof
PROFILE_FILE = 'vmprof_{}.dat'.format(os.getpid())
outfd = os.open(PROFILE_FILE, os.O_RDWR | os.O_CREAT | os.O_TRUNC)
vmprof.enable(outfd, period=0.01)
log.info("Entering event loop...")
reactor.run()
vmprof.disable()
else:
log.debug("Entering event loop...")
reactor.run()
except Exception as e:
log.info("Unhandled exception: {e}", e=e)
if reactor.running:
reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1)
reactor.stop()
else:
sys.exit(1)
def run():
"""
Entry point into (native) worker processes. This wires up stuff such that
a worker instance is talking WAMP-over-stdio to the node controller.
"""
import os
import sys
import platform
# create the top-level parser
#
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('-d',
'--debug',
action='store_true',
help='Debug on (optional).')
parser.add_argument('--reactor',
default=None,
choices=['select', 'poll', 'epoll', 'kqueue', 'iocp'],
help='Explicit Twisted reactor selection (optional).')
parser.add_argument('-c',
'--cbdir',
type=str,
help="Crossbar.io node directory (required).")
parser.add_argument('-n',
'--node',
type=str,
help='Crossbar.io node ID (required).')
parser.add_argument('-w',
'--worker',
type=str,
help='Crossbar.io worker ID (required).')
parser.add_argument('-r',
'--realm',
type=str,
help='Crossbar.io node (management) realm (required).')
parser.add_argument('-t',
'--type',
choices=['router', 'container'],
help='Worker type (required).')
parser.add_argument('--title',
type=str,
default=None,
help='Worker process title to set (optional).')
options = parser.parse_args()
# make sure logging to something else than stdio is setup _first_
#
from twisted.python import log
from crossbar.twisted.processutil import BareFormatFileLogObserver
flo = BareFormatFileLogObserver(sys.stderr)
log.startLoggingWithObserver(flo.emit)
try:
import setproctitle
except ImportError:
log.msg("Warning: could not set worker process title (setproctitle not installed)")
else:
# set process title if requested to
#
if options.title:
setproctitle.setproctitle(options.title)
else:
WORKER_TYPE_TO_TITLE = {
'router': 'crossbar-worker [router]',
'container': 'crossbar-worker [container]'
}
setproctitle.setproctitle(WORKER_TYPE_TO_TITLE[options.type].strip())
# we use an Autobahn utility to import the "best" available Twisted reactor
#
from autobahn.twisted.choosereactor import install_reactor
reactor = install_reactor(options.reactor)
from twisted.python.reflect import qual
log.msg("Running under {} using {} reactor".format(platform.python_implementation(), qual(reactor.__class__).split('.')[-1]))
options.cbdir = os.path.abspath(options.cbdir)
os.chdir(options.cbdir)
# log.msg("Starting from node directory {}".format(options.cbdir))
from crossbar.worker.router import RouterWorkerSession
from crossbar.worker.container import ContainerWorkerSession
WORKER_TYPE_TO_CLASS = {
'router': RouterWorkerSession,
'container': ContainerWorkerSession
}
from autobahn.twisted.websocket import WampWebSocketServerProtocol
class WorkerServerProtocol(WampWebSocketServerProtocol):
def connectionLost(self, reason):
try:
# this log message is unlikely to reach the controller (unless
# only stdin/stdout pipes were lost, but not stderr)
log.msg("Connection to node controller lost.")
WampWebSocketServerProtocol.connectionLost(self, reason)
except:
pass
finally:
# loosing the connection to the node controller is fatal:
# stop the reactor and exit with error
if reactor.running:
reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1)
reactor.stop()
else:
sys.exit(1)
try:
# create a WAMP application session factory
#
from autobahn.twisted.wamp import ApplicationSessionFactory
from autobahn.wamp.types import ComponentConfig
session_config = ComponentConfig(realm=options.realm, extra=options)
session_factory = ApplicationSessionFactory(session_config)
session_factory.session = WORKER_TYPE_TO_CLASS[options.type]
# create a WAMP-over-WebSocket transport server factory
#
from autobahn.twisted.websocket import WampWebSocketServerFactory
transport_factory = WampWebSocketServerFactory(session_factory, "ws://localhost", debug=False, debug_wamp=False)
transport_factory.protocol = WorkerServerProtocol
transport_factory.setProtocolOptions(failByDrop=False)
# create a protocol instance and wire up to stdio
#
from twisted.internet import stdio
proto = transport_factory.buildProtocol(None)
stdio.StandardIO(proto)
# now start reactor loop
#
log.msg("Entering event loop ..")
reactor.run()
except Exception as e:
log.msg("Unhandled exception: {}".format(e))
if reactor.running:
reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1)
reactor.stop()
else:
sys.exit(1)
##
from autobahn.wamp import types
session_factory.add(SessionKlass(types.ComponentConfig(realm = args.realm)))
if args.transport == "websocket":
## create a WAMP-over-WebSocket transport server factory with longpoll fallback
##
from autobahn.wamp.serializer import JsonSerializer, MsgPackSerializer
from autobahn.twisted.websocket import WampWebSocketServerFactory
from autobahn.twisted.resource import WebSocketResource
from twisted.web.server import Site
from twisted.web.static import File
from autobahn.wamp.http import WampHttpResource
ws_factory = WampWebSocketServerFactory(session_factory, debug_wamp = args.debug)
ws_factory.setProtocolOptions(failByDrop = False)
serializers = [MsgPackSerializer(), JsonSerializer()]
resource = WampHttpResource(serializers, debug=True, timeout=100, killAfter=120)
resource.factory = ws_factory
root = File("longpoll")
root.putChild("ws", WebSocketResource(ws_factory))
root.putChild("longpoll", resource)
if args.static_dir:
#root.putChild("web", File("/Users/arno/Documents/PycharmProjects/AutobahnJS/test/"))
#root.putChild("autobahn.js", File("/Users/arno/Documents/PycharmProjects/AutobahnJS/build/autobahn.js"))
#root.putChild("lib", File("/Users/arno/Documents/PycharmProjects/AutobahnJS/package/lib/"))
root.putChild("static", File(args.static_dir))
transport_factory = Site(root)
# create a WAMP router session factory
##
from autobahn.twisted.wamp import RouterSessionFactory
session_factory = RouterSessionFactory(router_factory)
session_factory.session = MyRouterSession
# start an embedded application component ..
##
component_config = types.ComponentConfig(realm="realm1")
component_session = TimeService(component_config)
session_factory.add(component_session)
# create a WAMP-over-WebSocket transport server factory
##
from autobahn.twisted.websocket import WampWebSocketServerFactory
transport_factory = WampWebSocketServerFactory(session_factory, args.wsurl, debug=False, debug_wamp=args.debug)
transport_factory.protocol = ServerProtocol
transport_factory._cookies = {}
transport_factory.setProtocolOptions(failByDrop=False)
from twisted.web.server import Site
from twisted.web.static import File
from autobahn.twisted.resource import WebSocketResource
# we serve static files under "/" ..
root = File(".")
# .. and our WebSocket server under "/ws"
resource = WebSocketResource(transport_factory)
root.putChild("ws", resource)
def run():
import sys, argparse
from twisted.python import log
from twisted.internet.endpoints import serverFromString
## parse command line arguments
##
def_wsocket = 'ws://127.0.0.1:8080/ws'
def_realm = 'realm1'
def_topic_base = 'sys'
def_dsn = 'dbname=autobahn host=localhost user=autouser'
def_endpoint='tcp:8080'
def_engine = 'PG9_4'
p = argparse.ArgumentParser(description="basicrouter example with database")
p.add_argument('-w', '--websocket', action='store', dest='wsocket', default=def_wsocket,
help='web socket '+def_wsocket)
p.add_argument('-r', '--realm', action='store', dest='realm', default=def_realm,
help='connect to websocket using "realm", default '+def_realm)
p.add_argument('-v', '--verbose', action='store_true', dest='verbose',
default=False, help='Verbose logging for debugging')
p.add_argument('--debug', action='store_true', dest='debug',
default=False, help='Autobahn layer debugging')
p.add_argument('-e', '--engine', action='store', dest='engine', default=def_engine,
help='if specified, a database engine will be attached. Note engine is rooted on --topic')
p.add_argument("--endpoint", type = str, default = "tcp:8080",
help = 'Twisted server endpoint descriptor, e.g. "tcp:8080" or "unix:/tmp/mywebsocket", default is "' + def_endpoint + '"')
p.add_argument('-d', '--dsn', action='store', dest='dsn', default=def_dsn,
help='if specified the database in dsn will be connected and ready')
p.add_argument('-t', '--topic', action='store', dest='topic_base', default=def_topic_base,
help='if you specify --dsn then you will need a topic to root it on, the default ' + def_topic_base + ' is fine.')
args = p.parse_args()
if args.verbose:
log.startLogging(sys.stdout)
## we use an Autobahn utility to install the "best" available Twisted reactor
##
from autobahn.twisted.choosereactor import install_reactor
reactor = install_reactor()
log.msg("Running on reactor {}".format(reactor))
# database workers...
userdb = UserDb(topic_base=args.topic_base+'.db',debug=args.verbose)
sessiondb = SessionDb(topic_base=args.topic_base,debug=args.verbose)
## create a WAMP router factory
##
component_config = types.ComponentConfig(realm = args.realm)
from autobahn.twisted.wamp import RouterFactory
router_factory = RouterFactory()
authorization_session = AuthorizeSession(component_config,
topic_base=args.topic_base+'.db',debug=args.verbose,db=sessiondb,router=AuthorizeRouter)
router_factory.router = authorization_session.ret_func
## create a WAMP router session factory
##
from autobahn.twisted.wamp import RouterSessionFactory
session_factory = RouterSessionFactory(router_factory)
session_factory.session = MyRouterSession
log.msg("session_factory.session")
session_factory.userdb = userdb
session_factory.sessiondb = sessiondb
log.msg("userdb, sessiondb")
sessiondb_component = SessionData(component_config,session_factory.sessiondb,
topic_base=args.topic_base)
session_factory.add(sessiondb_component)
session_factory.add(authorization_session)
log.msg("session_factory")
db_session = DB(component_config, engine=args.engine,
topic_base=args.topic_base+'.db', dsn=args.dsn, debug=args.verbose)
session_factory.add(db_session)
session_factory.userdb.set_session(db_session)
session_factory.sessiondb.set_session(db_session)
## create a WAMP-over-WebSocket transport server factory
##
from autobahn.twisted.websocket import WampWebSocketServerFactory
transport_factory = WampWebSocketServerFactory(session_factory, debug = args.debug)
transport_factory.setProtocolOptions(failByDrop = False)
## start the server from an endpoint
##
## this address clash detection was a goody I got from stackoverflow:
## http://stackoverflow.com/questions/12007316/exiting-twisted-application-after-listenfailure
server = serverFromString(reactor, args.endpoint)
def listen():
srv = server.listen(transport_factory)
def ListenFailed(reason):
log.msg("On Startup Listen Failed with {}".format(reason))
reactor.stop()
srv.addErrback(ListenFailed)
def addsession():
log.msg("here are three sessions {} {} {}".format(authorization_session, sessiondb_component, db_session))
qv = {
"sessiondb_component":sessiondb_component._session_id,
"db_session":db_session._session_id,
"authorization_session":authorization_session._session_id
}
session_factory.sessiondb.set_system_sessions(qv)
session_factory.sessiondb.add(0, sessiondb_component._session_id, sessiondb_component)
session_factory.sessiondb.add(0, db_session._session_id, db_session)
session_factory.sessiondb.add(0, authorization_session._session_id, authorization_session)
reactor.callWhenRunning(listen)
reactor.callWhenRunning(addsession)
reactor.run()
config.all_done = []
for C in embedded_components:
one_done = Deferred()
config.all_done.append(one_done)
c = C(config, one_done)
config.components.append(c)
session_factory.add(c)
if args.transport == "websocket":
## create a WAMP-over-WebSocket transport server factory
##
from autobahn.twisted.websocket import WampWebSocketServerFactory
transport_factory = WampWebSocketServerFactory(session_factory, debug_wamp = args.debug)
transport_factory.setProtocolOptions(failByDrop = False)
elif args.transport in ['rawsocket-json', 'rawsocket-msgpack']:
## create a WAMP-over-RawSocket transport server factory
##
if args.transport == 'rawsocket-msgpack':
from autobahn.wamp.serializer import MsgPackSerializer
serializer = MsgPackSerializer()
elif args.transport == 'rawsocket-json':
from autobahn.wamp.serializer import JsonSerializer
serializer = JsonSerializer()
else:
raise Exception("should not arrive here")
def run():
"""
Entry point into background worker process. This wires up stuff such that
a WorkerProcess instance is talking WAMP over stdio to the node controller.
"""
## create the top-level parser
##
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('-d',
'--debug',
action = 'store_true',
help = 'Debug on.')
parser.add_argument('--reactor',
default = None,
choices = ['select', 'poll', 'epoll', 'kqueue', 'iocp'],
help = 'Explicit Twisted reactor selection')
parser.add_argument('--cbdir',
type = str,
default = None,
help = "Crossbar.io node directory (overrides ${CROSSBAR_DIR} and the default ./.crossbar)")
parser.add_argument('-l',
'--logfile',
default = None,
help = 'Log to log file instead of stderr.')
options = parser.parse_args()
## make sure logging to something else than stdio is setup _first_
##
if options.logfile:
log.startLogging(open(options.logfile, 'a'))
else:
log.startLogging(sys.stderr)
pid = os.getpid()
## Crossbar.io node directory
##
if hasattr(options, 'cbdir') and not options.cbdir:
if os.environ.has_key("CROSSBAR_DIR"):
options.cbdir = os.environ['CROSSBAR_DIR']
else:
options.cbdir = '.crossbar'
options.cbdir = os.path.abspath(options.cbdir)
## we use an Autobahn utility to import the "best" available Twisted reactor
##
from autobahn.twisted.choosereactor import install_reactor
reactor = install_reactor(options.reactor)
##
from twisted.python.reflect import qual
log.msg("Worker {}: starting at node directory {} on {} ..".format(pid, options.cbdir, qual(reactor.__class__).split('.')[-1]))
try:
## create a WAMP application session factory
##
from autobahn.twisted.wamp import ApplicationSessionFactory
session_factory = ApplicationSessionFactory()
session_factory.options = options
session_factory.session = WorkerProcess
## create a WAMP-over-WebSocket transport server factory
##
from autobahn.twisted.websocket import WampWebSocketServerFactory
transport_factory = WampWebSocketServerFactory(session_factory, "ws://localhost", debug = False)
transport_factory.setProtocolOptions(failByDrop = False)
## create a protocol instance and wire up to stdio
##
from twisted.internet import stdio
proto = transport_factory.buildProtocol(None)
stdio.StandardIO(proto)
## now start reactor loop
##
log.msg("Worker {}: entering event loop ..".format(pid))
reactor.run()
except Exception as e:
log.msg("Worker {}: Unhandled exception - {}".format(pid, e))
raise e
sys.exit(1)
def _run_command_exec_worker(options, reactor=None, personality=None):
"""
Entry point into (native) worker processes. This wires up stuff such that
a worker instance is talking WAMP-over-stdio to the node controller.
"""
import os
import sys
import platform
import signal
# https://coverage.readthedocs.io/en/coverage-4.4.2/subprocess.html#measuring-sub-processes
MEASURING_COVERAGE = False
if 'COVERAGE_PROCESS_START' in os.environ:
try:
import coverage
except ImportError:
pass
else:
# The following will read the environment variable COVERAGE_PROCESS_START,
# and that should be set to the .coveragerc file:
#
# export COVERAGE_PROCESS_START=${PWD}/.coveragerc
#
coverage.process_startup()
MEASURING_COVERAGE = True
# we use an Autobahn utility to import the "best" available Twisted reactor
from autobahn.twisted.choosereactor import install_reactor
reactor = install_reactor(options.reactor)
# make sure logging to something else than stdio is setup _first_
from crossbar._logging import make_JSON_observer, cb_logging_aware
from txaio import make_logger, start_logging
from twisted.logger import globalLogPublisher
from twisted.python.reflect import qual
log = make_logger()
# Print a magic phrase that tells the capturing logger that it supports
# Crossbar's rich logging
print(cb_logging_aware, file=sys.__stderr__)
sys.__stderr__.flush()
flo = make_JSON_observer(sys.__stderr__)
globalLogPublisher.addObserver(flo)
# Ignore SIGINT so we get consistent behavior on control-C versus
# sending SIGINT to the controller process. When the controller is
# shutting down, it sends TERM to all its children but ctrl-C
# handling will send a SIGINT to all the processes in the group
# (so then the controller sends a TERM but the child already or
# will very shortly get a SIGINT as well). Twisted installs signal
# handlers, but not for SIGINT if there's already a custom one
# present.
def ignore(sig, frame):
log.debug("Ignoring SIGINT in worker.")
signal.signal(signal.SIGINT, ignore)
# actually begin logging
start_logging(None, options.loglevel)
# get personality klass, eg "crossbar.personality.Personality"
l = options.personality.split('.')
personality_module, personality_klass = '.'.join(l[:-1]), l[-1]
# now load the personality module and class
_mod = importlib.import_module(personality_module)
Personality = getattr(_mod, personality_klass)
# get worker klass, eg "crossbar.worker.container.ContainerController"
l = options.klass.split('.')
worker_module, worker_klass = '.'.join(l[:-1]), l[-1]
# now load the worker module and class
_mod = importlib.import_module(worker_module)
klass = getattr(_mod, worker_klass)
log.info(
'Starting worker "{worker_id}" for node "{node_id}" with personality "{personality}" {worker_class}',
worker_id=options.worker,
node_id=options.node,
personality=Personality.NAME,
worker_class=hltype(klass),
)
log.info(
'Running as PID {pid} on {python}-{reactor}',
pid=os.getpid(),
python=platform.python_implementation(),
reactor=qual(reactor.__class__).split('.')[-1],
)
if MEASURING_COVERAGE:
log.info(hl('Code coverage measurements enabled (coverage={coverage_version}).', color='green', bold=True),
coverage_version=coverage.__version__)
# set process title if requested to
#
try:
import setproctitle
except ImportError:
log.debug("Could not set worker process title (setproctitle not installed)")
else:
if options.title:
setproctitle.setproctitle(options.title)
else:
setproctitle.setproctitle('crossbar-worker [{}]'.format(options.klass))
# node directory
#
options.cbdir = os.path.abspath(options.cbdir)
os.chdir(options.cbdir)
# log.msg("Starting from node directory {}".format(options.cbdir))
# set process title if requested to
#
try:
import setproctitle
except ImportError:
log.debug("Could not set worker process title (setproctitle not installed)")
else:
if options.title:
setproctitle.setproctitle(options.title)
else:
setproctitle.setproctitle(
'crossbar-worker [{}]'.format(options.klass)
)
from twisted.internet.error import ConnectionDone
from autobahn.twisted.websocket import WampWebSocketServerProtocol
class WorkerServerProtocol(WampWebSocketServerProtocol):
def connectionLost(self, reason):
# the behavior here differs slightly whether we're shutting down orderly
# or shutting down because of "issues"
if isinstance(reason.value, ConnectionDone):
was_clean = True
else:
was_clean = False
try:
# this log message is unlikely to reach the controller (unless
# only stdin/stdout pipes were lost, but not stderr)
if was_clean:
log.info("Connection to node controller closed cleanly")
else:
log.warn("Connection to node controller lost: {reason}", reason=reason)
# give the WAMP transport a change to do it's thing
WampWebSocketServerProtocol.connectionLost(self, reason)
except:
# we're in the process of shutting down .. so ignore ..
pass
finally:
# after the connection to the node controller is gone,
# the worker is "orphane", and should exit
# determine process exit code
if was_clean:
exit_code = 0
else:
exit_code = 1
# exit the whole worker process when the reactor has stopped
reactor.addSystemEventTrigger('after', 'shutdown', os._exit, exit_code)
# stop the reactor
try:
reactor.stop()
except ReactorNotRunning:
pass
try:
# define a WAMP application session factory
#
from autobahn.wamp.types import ComponentConfig
def make_session():
session_config = ComponentConfig(realm=options.realm, extra=options)
session = klass(config=session_config, reactor=reactor, personality=Personality)
return session
# create a WAMP-over-WebSocket transport server factory
#
from autobahn.twisted.websocket import WampWebSocketServerFactory
transport_factory = WampWebSocketServerFactory(make_session, u'ws://localhost')
transport_factory.protocol = WorkerServerProtocol
transport_factory.setProtocolOptions(failByDrop=False)
# create a protocol instance and wire up to stdio
#
from twisted.python.runtime import platform as _platform
from twisted.internet import stdio
proto = transport_factory.buildProtocol(None)
if _platform.isWindows():
stdio.StandardIO(proto)
else:
stdio.StandardIO(proto, stdout=3)
# now start reactor loop
#
if False:
log.info("vmprof enabled.")
import os
import vmprof
PROFILE_FILE = 'vmprof_{}.dat'.format(os.getpid())
outfd = os.open(PROFILE_FILE, os.O_RDWR | os.O_CREAT | os.O_TRUNC)
vmprof.enable(outfd, period=0.01)
log.info(hl('Entering event reactor ...', color='cyan', bold=True))
reactor.run()
vmprof.disable()
else:
log.info(hl('Entering event reactor ...', color='cyan', bold=True))
reactor.run()
except Exception as e:
log.info("Unhandled exception: {e}", e=e)
if reactor.running:
reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1)
reactor.stop()
else:
sys.exit(1)
if __name__ == '__main__':
## 0) start logging to console
log.startLogging(sys.stdout)
## 1) create a WAMP router factory
router_factory = RouterFactory()
## 2) create a WAMP router session factory
session_factory = RouterSessionFactory(router_factory)
## 3) Optionally, add embedded WAMP application sessions to the router
session_factory.add(MyBackendComponent())
## 4) create a WAMP-over-WebSocket transport server factory
transport_factory1 = WampWebSocketServerFactory(session_factory,
debug=False)
## 5) start the server from a Twisted endpoint
server1 = serverFromString(reactor, "tcp:8080")
server1.listen(transport_factory1)
## 6) create a WAMP-over-RawSocket-MsgPack transport server factory
serializer = MsgPackSerializer()
serializer._serializer.ENABLE_V5 = False
transport_factory2 = WampRawSocketServerFactory(session_factory,
serializer,
debug=True)
## 7) start the server from a Twisted endpoint
server2 = serverFromString(reactor, "tcp:8090")
server2.listen(transport_factory2)
yield self.register(Sketch_Stop, u"allofw.sketch.stop")
yield self.register(Sketch_List, u"allofw.sketch.list")
yield self.register(Sketch_GetCode, u"allofw.sketch.get_code")
yield self.register(Project_List, u"allofw.project.list")
yield self.register(Project_Delete, u"allofw.project.delete")
yield self.register(Project_UploadFile, u"allofw.project.upload_file")
yield self.register(Project_RenameFile, u"allofw.project.rename_file")
yield self.register(Project_ListFiles, u"allofw.project.list_files")
yield self.register(Project_LoadProject,
u"allofw.project.load_project")
yield self.register(Project_SaveProject,
u"allofw.project.save_project")
root = File("static")
session_factory = RouterSessionFactory(RouterFactory())
component_config = types.ComponentConfig(realm="anonymous")
session_factory.add(AllofwServer(component_config))
factory = WampWebSocketServerFactory(session_factory)
factory.startFactory()
resource = WebSocketResource(factory)
root.putChild("ws", resource)
site = Site(root)
reactor.listenTCP(int(config['webserver']['port']),
site,
interface=config['webserver']['listen'])
reactor.run()
class Node:
"""
A Crossbar.io node is the running a controller process
and one or multiple worker processes.
A single Crossbar.io node runs exactly one instance of
this class, hence this class can be considered a system
singleton.
"""
def __init__(self, reactor, cbdir, debug = False):
"""
Ctor.
:param reactor: Reactor to run on.
:type reactor: obj
:param cbdir: Crossbar.io node directory to run from.
:type cbdir: str
"""
self.debug = debug
self._reactor = reactor
self._cbdir = cbdir
self._worker_processes = {}
## node name: FIXME
self._node_name = "{}-{}".format(socket.getfqdn(), os.getpid())
self._node_name.replace('-', '_')
self._node_name = '918234'
## node management
self._management_url = "ws://127.0.0.1:7000"
#self._management_url = "wss://cloud.crossbar.io"
self._management_realm = "crossbar.cloud.aliceblue"
## load Crossbar.io node configuration
##
cf = os.path.join(self._cbdir, 'config.json')
with open(cf, 'rb') as infile:
self._config = json.load(infile)
def start(self):
"""
Starts this node. This will start a node controller
and then spawn new worker processes as needed.
The node controller will watch spawned processes,
communicate via stdio with the worker, and start
and restart the worker processes as needed.
"""
## the node controller singleton WAMP application session
##
self._node_controller_session = NodeControllerSession(self)
## router and factory that creates router sessions
##
self._router_factory = RouterFactory()
self._router_session_factory = RouterSessionFactory(self._router_factory)
## add the node controller singleton session to the router
##
self._router_session_factory.add(self._node_controller_session)
if True:
## create a WAMP-over-WebSocket transport server factory
##
from autobahn.twisted.websocket import WampWebSocketServerFactory
from twisted.internet.endpoints import serverFromString
self._router_server_transport_factory = WampWebSocketServerFactory(self._router_session_factory, "ws://localhost:9000", debug = False)
self._router_server_transport_factory.setProtocolOptions(failByDrop = False)
## start the WebSocket server from an endpoint
##
self._router_server = serverFromString(self._reactor, "tcp:9000")
self._router_server.listen(self._router_server_transport_factory)
## factory that creates router session transports. these are for clients
## that talk WAMP-WebSocket over pipes with spawned worker processes and
## for any uplink session to a management service
##
self._router_client_transport_factory = WampWebSocketClientFactory(self._router_session_factory, "ws://localhost", debug = False)
self._router_client_transport_factory.setProtocolOptions(failByDrop = False)
if False:
management_session_factory = ApplicationSessionFactory()
management_session_factory.session = NodeManagementSession
management_session_factory.node_controller_session = node_controller_session
management_transport_factory = WampWebSocketClientFactory(management_session_factory, "ws://127.0.0.1:7000")
management_transport_factory.setProtocolOptions(failByDrop = False)
management_client = clientFromString(self._reactor, "tcp:127.0.0.1:7000")
management_client.connect(management_transport_factory)
## startup the node from configuration file
##
self._node_controller_session.run_node_config(self._config)
config.all_done = []
for C in embedded_components:
one_done = Deferred()
config.all_done.append(one_done)
c = C(config, one_done)
config.components.append(c)
session_factory.add(c)
if args.transport == "websocket":
## create a WAMP-over-WebSocket transport server factory
##
from autobahn.twisted.websocket import WampWebSocketServerFactory
transport_factory = WampWebSocketServerFactory(session_factory,
debug_wamp=args.debug)
transport_factory.setProtocolOptions(failByDrop=False)
elif args.transport in ['rawsocket-json', 'rawsocket-msgpack']:
## create a WAMP-over-RawSocket transport server factory
##
if args.transport == 'rawsocket-msgpack':
from autobahn.wamp.serializer import MsgPackSerializer
serializer = MsgPackSerializer()
elif args.transport == 'rawsocket-json':
from autobahn.wamp.serializer import JsonSerializer
serializer = JsonSerializer()
else:
raise Exception("should not arrive here")
def run(pm, reactor):
from sputnik.webserver.router.twisted.wamp import RouterFactory
router_factory = RouterFactory()
router_factory.router = SputnikRouter
router_factory.authz_plugins = \
pm.services.get("sputnik.webserver.plugins.authz", [])
router_factory.schema_plugins = \
pm.services.get("sputnik.webserver.plugins.schema", [])
from sputnik.webserver.router.twisted.wamp import RouterSessionFactory
session_factory = RouterSessionFactory(router_factory)
session_factory.session = SputnikRouterSession
authn_stack = [("ip.IPFilter", "requisite"),
("anonymous.AnonymousLogin", "sufficient"),
("cookie.CookieLogin", "sufficient"),
("wampcra.WAMPCRALogin", "requisite"),
("totp.TOTPVerification", "requisite")]
session_factory.plugins = []
for plugin_name, flag in authn_stack:
path = "sputnik.webserver.plugins.authn." + plugin_name
session_factory.plugins.append((pm.plugins[path], flag))
rpc_plugins = pm.services.get("sputnik.webserver.plugins.rpc", [])
feeds_plugins = pm.services.get("sputnik.webserver.plugins.feeds", [])
svc_plugins = rpc_plugins + feeds_plugins
for plugin in svc_plugins:
component_session = plugin
component_session.config.realm = u"sputnik"
session_factory.add(component_session,
plugin.plugin_path.decode("ascii"), u"trusted")
uri = "ws://"
if config.getboolean("webserver", "ssl"):
uri = "wss://"
address = config.get("webserver", "ws_address")
port = config.getint("webserver", "ws_port")
uri += "%s:%s/" % (address, port)
from autobahn.twisted.websocket import WampWebSocketServerFactory
transport_factory = WampWebSocketServerFactory(session_factory,
uri,
debug=False,
debug_wamp=False)
transport_factory.setProtocolOptions(failByDrop=False)
watchdog(config.get("watchdog", "webserver"))
from twisted.web.server import Site
from autobahn.twisted.resource import WebSocketResource
root = Root()
ws_resource = WebSocketResource(transport_factory)
rest_resource = pm.plugins['sputnik.webserver.rest.RESTProxy']
root.putChild("ws", ws_resource)
root.putChild("api", rest_resource)
site = Site(root)
site.noisy = False
site.log = lambda _: None
from twisted.internet.endpoints import serverFromString, quoteStringArgument
if config.getboolean("webserver", "ssl"):
key = config.get("webserver", "ssl_key")
cert = config.get("webserver", "ssl_cert")
cert_chain = config.get("webserver", "ssl_cert_chain")
# TODO: Add dhparameters
# See https://twistedmatrix.com/documents/14.0.0/core/howto/endpoints.html
server = serverFromString(
reactor,
b"ssl:%d:privateKey=%s:certKey=%s:extraCertChain=%s:sslmethod=TLSv1_METHOD"
% (port, quoteStringArgument(key), quoteStringArgument(cert),
quoteStringArgument(cert_chain)))
else:
server = serverFromString(reactor, b"tcp:%d" % port)
server.listen(site)
def run():
"""
Entry point into background worker process. This wires up stuff such that
a WorkerProcess instance is talking WAMP over stdio to the node controller.
"""
## create the top-level parser
##
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('-d',
'--debug',
action = 'store_true',
help = 'Debug on.')
parser.add_argument('--reactor',
default = None,
choices = ['select', 'poll', 'epoll', 'kqueue', 'iocp'],
help = 'Explicit Twisted reactor selection')
parser.add_argument('--cbdir',
type = str,
default = None,
help = "Crossbar.io node directory (overrides ${CROSSBAR_DIR} and the default ./.crossbar)")
parser.add_argument('-n',
'--name',
default = None,
help = 'Optional process name to set.')
options = parser.parse_args()
## make sure logging to something else than stdio is setup _first_
##
from crossbar.twisted.process import BareFormatFileLogObserver
flo = BareFormatFileLogObserver(sys.stderr)
log.startLoggingWithObserver(flo.emit)
## the worker's PID
##
pid = os.getpid()
try:
import setproctitle
except ImportError:
log.msg("Warning, could not set process title (setproctitle not installed)")
else:
## set process title if requested to
##
if options.name:
setproctitle.setproctitle(options.name)
else:
setproctitle.setproctitle("Crossbar.io Worker")
## Crossbar.io node directory
##
if hasattr(options, 'cbdir') and not options.cbdir:
if os.environ.has_key("CROSSBAR_DIR"):
options.cbdir = os.environ['CROSSBAR_DIR']
else:
options.cbdir = '.crossbar'
options.cbdir = os.path.abspath(options.cbdir)
log.msg("Starting from node directory {}.".format(options.cbdir))
## we use an Autobahn utility to import the "best" available Twisted reactor
##
from autobahn.twisted.choosereactor import install_reactor
reactor = install_reactor(options.reactor)
from twisted.python.reflect import qual
log.msg("Running on {} reactor.".format(qual(reactor.__class__).split('.')[-1]))
from autobahn.twisted.websocket import WampWebSocketServerProtocol
class WorkerServerProtocol(WampWebSocketServerProtocol):
def connectionLost(self, reason):
try:
log.msg("Connection to node controller lost.")
WampWebSocketServerProtocol.connectionLost(self, reason)
except:
pass
finally:
## loosing the connection to the node controller (the pipes) is fatal.
## stop the reactor and exit with error
if reactor.running:
reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1)
reactor.stop()
else:
sys.exit(1)
try:
## create a WAMP application session factory
##
from autobahn.twisted.wamp import ApplicationSessionFactory
session_factory = ApplicationSessionFactory()
session_factory.options = options
session_factory.session = WorkerProcess
## create a WAMP-over-WebSocket transport server factory
##
from autobahn.twisted.websocket import WampWebSocketServerFactory
transport_factory = WampWebSocketServerFactory(session_factory, "ws://localhost", debug = False)
transport_factory.protocol = WorkerServerProtocol
transport_factory.setProtocolOptions(failByDrop = False)
## create a protocol instance and wire up to stdio
##
from twisted.internet import stdio
proto = transport_factory.buildProtocol(None)
stdio.StandardIO(proto)
## now start reactor loop
##
log.msg("Entering event loop ..")
#reactor.callLater(4, reactor.stop)
reactor.run()
except Exception as e:
log.msg("Unhandled exception - {}".format(e))
if reactor.running:
reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1)
reactor.stop()
else:
sys.exit(1)
def run():
"""
Entry point into (native) worker processes. This wires up stuff such that
a worker instance is talking WAMP-over-stdio to the node controller.
"""
import os
import sys
import platform
import signal
# Ignore SIGINT so we get consistent behavior on control-C versus
# sending SIGINT to the controller process. When the controller is
# shutting down, it sends TERM to all its children but ctrl-C
# handling will send a SIGINT to all the processes in the group
# (so then the controller sends a TERM but the child already or
# will very shortly get a SIGINT as well). Twisted installs signal
# handlers, but not for SIGINT if there's already a custom one
# present.
def ignore(sig, frame):
log.debug("Ignoring SIGINT in worker.")
signal.signal(signal.SIGINT, ignore)
# create the top-level parser
#
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('-d',
'--debug',
action='store_true',
help='Debug on (optional).')
parser.add_argument('--reactor',
default=None,
choices=['select', 'poll', 'epoll', 'kqueue', 'iocp'],
help='Explicit Twisted reactor selection (optional).')
parser.add_argument('-c',
'--cbdir',
type=str,
help="Crossbar.io node directory (required).")
parser.add_argument('-n',
'--node',
type=str,
help='Crossbar.io node ID (required).')
parser.add_argument('-w',
'--worker',
type=str,
help='Crossbar.io worker ID (required).')
parser.add_argument('-r',
'--realm',
type=str,
help='Crossbar.io node (management) realm (required).')
parser.add_argument('-t',
'--type',
choices=['router', 'container'],
help='Worker type (required).')
parser.add_argument('--title',
type=str,
default=None,
help='Worker process title to set (optional).')
options = parser.parse_args()
# make sure logging to something else than stdio is setup _first_
#
from twisted.logger import globalLogBeginner
from crossbar._logging import Logger, make_JSON_observer
log = Logger()
_stderr = sys.stderr
flo = make_JSON_observer(_stderr)
globalLogBeginner.beginLoggingTo([flo])
try:
import setproctitle
except ImportError:
log.info("Warning: could not set worker process title (setproctitle not installed)")
else:
# set process title if requested to
#
if options.title:
setproctitle.setproctitle(options.title)
else:
WORKER_TYPE_TO_TITLE = {
'router': 'crossbar-worker [router]',
'container': 'crossbar-worker [container]'
}
setproctitle.setproctitle(WORKER_TYPE_TO_TITLE[options.type].strip())
# we use an Autobahn utility to import the "best" available Twisted reactor
#
from autobahn.twisted.choosereactor import install_reactor
reactor = install_reactor(options.reactor)
from twisted.python.reflect import qual
log.info("Running under {python} using {reactor} reactor",
python=platform.python_implementation(),
reactor=qual(reactor.__class__).split('.')[-1])
options.cbdir = os.path.abspath(options.cbdir)
os.chdir(options.cbdir)
# log.msg("Starting from node directory {}".format(options.cbdir))
from crossbar.worker.router import RouterWorkerSession
from crossbar.worker.container import ContainerWorkerSession
WORKER_TYPE_TO_CLASS = {
'router': RouterWorkerSession,
'container': ContainerWorkerSession
}
from autobahn.twisted.websocket import WampWebSocketServerProtocol
class WorkerServerProtocol(WampWebSocketServerProtocol):
def connectionLost(self, reason):
try:
# this log message is unlikely to reach the controller (unless
# only stdin/stdout pipes were lost, but not stderr)
log.warn("Connection to node controller lost.")
WampWebSocketServerProtocol.connectionLost(self, reason)
except:
pass
finally:
# loosing the connection to the node controller is fatal:
# stop the reactor and exit with error
if reactor.running:
reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1)
reactor.stop()
# if the reactor *isn't* running, we're already shutting down
try:
# create a WAMP application session factory
#
from autobahn.twisted.wamp import ApplicationSessionFactory
from autobahn.wamp.types import ComponentConfig
session_config = ComponentConfig(realm=options.realm, extra=options)
session_factory = ApplicationSessionFactory(session_config)
session_factory.session = WORKER_TYPE_TO_CLASS[options.type]
# create a WAMP-over-WebSocket transport server factory
#
from autobahn.twisted.websocket import WampWebSocketServerFactory
transport_factory = WampWebSocketServerFactory(session_factory, "ws://localhost", debug=False, debug_wamp=False)
transport_factory.protocol = WorkerServerProtocol
transport_factory.setProtocolOptions(failByDrop=False)
# create a protocol instance and wire up to stdio
#
from twisted.internet import stdio
proto = transport_factory.buildProtocol(None)
stdio.StandardIO(proto)
# now start reactor loop
#
log.info("Entering event loop...")
reactor.run()
except Exception as e:
log.info("Unhandled exception: {}".format(e))
if reactor.running:
reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1)
reactor.stop()
else:
sys.exit(1)
def __init__(self, factory, cbdir, config, templates):
"""
Ctor.
:param factory: WAMP session factory.
:type factory: An instance of ..
:param cbdir: The Crossbar.io node directory.
:type cbdir: str
:param config: Crossbar transport configuration.
:type config: dict
"""
self.debug = config.get('debug', False)
self.debug_traffic = config.get('debug_traffic', False)
options = config.get('options', {})
server = "Crossbar/{}".format(crossbar.__version__)
externalPort = options.get('external_port', None)
## explicit list of WAMP serializers
##
if 'serializers' in config:
serializers = []
sers = set(config['serializers'])
if 'json' in sers:
## try JSON WAMP serializer
try:
from autobahn.wamp.serializer import JsonSerializer
serializers.append(JsonSerializer())
except ImportError:
print("Warning: could not load WAMP-JSON serializer")
else:
sers.discard('json')
if 'msgpack' in sers:
## try MsgPack WAMP serializer
try:
from autobahn.wamp.serializer import MsgPackSerializer
serializers.append(MsgPackSerializer())
except ImportError:
print("Warning: could not load WAMP-MsgPack serializer")
else:
sers.discard('msgpack')
if not serializers:
raise Exception("no valid WAMP serializers specified")
if len(sers) > 0:
raise Exception(
"invalid WAMP serializers specified: {}".format(sers))
else:
serializers = None
WampWebSocketServerFactory.__init__(self,
factory,
serializers=serializers,
url=config.get('url', None),
server=server,
externalPort=externalPort,
debug=self.debug,
debug_wamp=self.debug)
## Crossbar.io node directory
self._cbdir = cbdir
## transport configuration
self._config = config
## Jinja2 templates for 404 etc
self._templates = templates
## cookie tracking
if 'cookie' in config:
if 'database' in config['cookie']:
dbfile = os.path.abspath(
os.path.join(self._cbdir, config['cookie']['database']))
self._cookiestore = PersistentCookieStore(
dbfile, config['cookie'])
log.msg("Persistent cookie store active: {}".format(dbfile))
else:
self._cookiestore = CookieStore(config['cookie'])
log.msg("Transient cookie store active.")
else:
self._cookiestore = None
## set WebSocket options
set_websocket_options(self, options)
from autobahn.wamp.router import RouterFactory
router_factory = RouterFactory()
## create a WAMP router session factory
##
from autobahn.twisted.wamp import RouterSessionFactory
session_factory = RouterSessionFactory(router_factory)
## [ ... ... ]
session_factory.add(ModelServer())
## create a WAMP-over-WebSocket transport server factory
##
from autobahn.twisted.websocket import WampWebSocketServerFactory
wamp_factory = WampWebSocketServerFactory(session_factory,
"ws://localhost:8080",
debug=True)
wamp_factory.setProtocolOptions(failByDrop=False)
webroot = pathjoin(PROJECT_ROOT, "src", "frontend")
assets = pathjoin(PROJECT_ROOT, "assets")
if debug:
print "putting", webroot, "at root"
print "putting", assets, "at assets"
## we serve static files (most of the frontend html, js, and css) under "/" ..
## except for some which are under assets/
## and we have our WebSocket server under "/ws"
root = File(webroot)
assets = File(assets)
data_resource = WebSocketResource(data_endpoint)
def run():
"""
Entry point into (native) worker processes. This wires up stuff such that
a worker instance is talking WAMP-over-stdio to the node controller.
"""
import os
import sys
import platform
import signal
# Ignore SIGINT so we get consistent behavior on control-C versus
# sending SIGINT to the controller process. When the controller is
# shutting down, it sends TERM to all its children but ctrl-C
# handling will send a SIGINT to all the processes in the group
# (so then the controller sends a TERM but the child already or
# will very shortly get a SIGINT as well). Twisted installs signal
# handlers, but not for SIGINT if there's already a custom one
# present.
def ignore(sig, frame):
log.debug("Ignoring SIGINT in worker.")
signal.signal(signal.SIGINT, ignore)
# create the top-level parser
#
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--reactor',
default=None,
choices=['select', 'poll', 'epoll', 'kqueue', 'iocp'],
help='Explicit Twisted reactor selection (optional).')
parser.add_argument('--loglevel',
default="info",
choices=['none', 'error', 'warn', 'info', 'debug', 'trace'],
help='Initial log level.')
parser.add_argument('-c',
'--cbdir',
type=str,
help="Crossbar.io node directory (required).")
parser.add_argument('-n',
'--node',
type=str,
help='Crossbar.io node ID (required).')
parser.add_argument('-w',
'--worker',
type=str,
help='Crossbar.io worker ID (required).')
parser.add_argument('-r',
'--realm',
type=str,
help='Crossbar.io node (management) realm (required).')
parser.add_argument('-t',
'--type',
choices=['router', 'container'],
help='Worker type (required).')
parser.add_argument('--title',
type=str,
default=None,
help='Worker process title to set (optional).')
options = parser.parse_args()
# make sure logging to something else than stdio is setup _first_
#
from crossbar._logging import make_JSON_observer, cb_logging_aware, _stderr
from crossbar._logging import make_logger, log_publisher, start_logging
from crossbar._logging import set_global_log_level
# Set the global log level
set_global_log_level(options.loglevel)
log = make_logger()
# Print a magic phrase that tells the capturing logger that it supports
# Crossbar's rich logging
print(cb_logging_aware, file=_stderr)
_stderr.flush()
flo = make_JSON_observer(_stderr)
log_publisher.addObserver(flo)
start_logging()
try:
import setproctitle
except ImportError:
log.debug("Could not set worker process title (setproctitle not installed)")
else:
# set process title if requested to
#
if options.title:
setproctitle.setproctitle(options.title)
else:
WORKER_TYPE_TO_TITLE = {
'router': 'crossbar-worker [router]',
'container': 'crossbar-worker [container]'
}
setproctitle.setproctitle(WORKER_TYPE_TO_TITLE[options.type].strip())
# we use an Autobahn utility to import the "best" available Twisted reactor
#
from autobahn.twisted.choosereactor import install_reactor
reactor = install_reactor(options.reactor)
from twisted.python.reflect import qual
log.info("Worker running under {python}-{reactor}",
python=platform.python_implementation(),
reactor=qual(reactor.__class__).split('.')[-1])
options.cbdir = os.path.abspath(options.cbdir)
os.chdir(options.cbdir)
# log.msg("Starting from node directory {}".format(options.cbdir))
from crossbar.worker.router import RouterWorkerSession
from crossbar.worker.container import ContainerWorkerSession
WORKER_TYPE_TO_CLASS = {
'router': RouterWorkerSession,
'container': ContainerWorkerSession
}
from autobahn.twisted.websocket import WampWebSocketServerProtocol
class WorkerServerProtocol(WampWebSocketServerProtocol):
def connectionLost(self, reason):
try:
# this log message is unlikely to reach the controller (unless
# only stdin/stdout pipes were lost, but not stderr)
log.warn("Connection to node controller lost.")
WampWebSocketServerProtocol.connectionLost(self, reason)
except:
pass
finally:
# losing the connection to the node controller is fatal:
# stop the reactor and exit with error
log.info("No more controller connection; shutting down.")
reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1)
try:
reactor.stop()
except ReactorNotRunning:
pass
try:
# create a WAMP application session factory
#
from autobahn.twisted.wamp import ApplicationSessionFactory
from autobahn.wamp.types import ComponentConfig
session_config = ComponentConfig(realm=options.realm, extra=options)
session_factory = ApplicationSessionFactory(session_config)
session_factory.session = WORKER_TYPE_TO_CLASS[options.type]
# create a WAMP-over-WebSocket transport server factory
#
from autobahn.twisted.websocket import WampWebSocketServerFactory
transport_factory = WampWebSocketServerFactory(session_factory, "ws://localhost", debug=False, debug_wamp=False)
transport_factory.protocol = WorkerServerProtocol
transport_factory.setProtocolOptions(failByDrop=False)
# create a protocol instance and wire up to stdio
#
from twisted.python.runtime import platform as _platform
from twisted.internet import stdio
proto = transport_factory.buildProtocol(None)
if _platform.isWindows():
stdio.StandardIO(proto)
else:
stdio.StandardIO(proto, stdout=3)
# now start reactor loop
#
if False:
log.info("vmprof enabled.")
import os
import vmprof
PROFILE_FILE = 'vmprof_{}.dat'.format(os.getpid())
outfd = os.open(PROFILE_FILE, os.O_RDWR | os.O_CREAT | os.O_TRUNC)
vmprof.enable(outfd, period=0.01)
log.info("Entering event loop...")
reactor.run()
vmprof.disable()
else:
log.debug("Entering event loop...")
reactor.run()
except Exception as e:
log.info("Unhandled exception: {}".format(e))
if reactor.running:
reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1)
reactor.stop()
else:
sys.exit(1)
def run():
import sys, argparse
from twisted.python import log
from twisted.internet.endpoints import serverFromString
## parse command line arguments
##
def_wsocket = 'ws://127.0.0.1:8080/ws'
def_xsocket = None
def_user = '******'
def_secret = '123test'
def_realm = 'realm1'
def_topic_base = 'sys'
def_dsn = 'dbname=autobahn host=localhost user=autouser'
def_endpoint = 'tcp:8080'
def_engine = 'PG9_4'
p = argparse.ArgumentParser(
description="basicrouter example with database")
p.add_argument('-w',
'--websocket',
action='store',
dest='wsocket',
default=def_wsocket,
help='web socket ' + def_wsocket)
p.add_argument('-x',
'--xsocket',
action='store',
dest='xsocket',
default=def_xsocket,
help='x socket definition, default is: None ')
p.add_argument('-r',
'--realm',
action='store',
dest='realm',
default=def_realm,
help='connect to websocket using "realm", default ' +
def_realm)
p.add_argument('-v',
'--verbose',
action='store_true',
dest='verbose',
default=False,
help='Verbose logging for debugging')
p.add_argument('-u',
'--user',
action='store',
dest='user',
default=def_user,
help='connect to websocket as user, default is: ' +
def_user)
p.add_argument('-s',
'--secret',
action='store',
dest='password',
default=def_secret,
help='users "secret" password')
p.add_argument('--debug',
action='store_true',
dest='debug',
default=False,
help='Autobahn layer debugging')
p.add_argument(
'-e',
'--engine',
action='store',
dest='engine',
default=def_engine,
help=
'if specified, a database engine will be attached. Note engine is rooted on --topic'
)
p.add_argument(
"--endpoint",
type=str,
default="tcp:8080",
help=
'Twisted server endpoint descriptor, e.g. "tcp:8080" or "unix:/tmp/mywebsocket", default is "'
+ def_endpoint + '"')
p.add_argument(
'-d',
'--dsn',
action='store',
dest='dsn',
default=def_dsn,
help='if specified the database in dsn will be connected and ready')
p.add_argument(
'-t',
'--topic',
action='store',
dest='topic_base',
default=def_topic_base,
help=
'if you specify --dsn then you will need a topic to root it on, the default '
+ def_topic_base + ' is fine.')
args = p.parse_args()
if args.verbose:
log.startLogging(sys.stdout)
## we use an Autobahn utility to install the "best" available Twisted reactor
##
from autobahn.twisted.choosereactor import install_reactor
reactor = install_reactor()
log.msg("Running on reactor {}".format(reactor))
# database workers...
userdb = UserDb(topic_base=args.topic_base + '.db', debug=args.verbose)
sessiondb = SessionDb(topic_base=args.topic_base, debug=args.verbose)
## create a WAMP router factory
##
component_config = types.ComponentConfig(realm=args.realm)
xomponent_config = types.ComponentConfig(realm=args.realm)
ai = {
'auth_type': 'wampcra',
'auth_user': args.user,
'auth_password': args.password
}
from autobahn.twisted.wamp import RouterFactory
router_factory = RouterFactory()
authorization_session = AuthorizeSession(component_config,
topic_base=args.topic_base +
'.db',
debug=args.verbose,
db=sessiondb,
router=AuthorizeRouter)
router_factory.router = authorization_session.ret_func
## create a WAMP router session factory
##
from autobahn.twisted.wamp import RouterSessionFactory
session_factory = RouterSessionFactory(router_factory)
session_factory.session = MyRouterSession
log.msg("session_factory.session")
session_factory.userdb = userdb
session_factory.sessiondb = sessiondb
log.msg("userdb, sessiondb")
sessiondb_component = SessionData(component_config,
session_factory.sessiondb,
topic_base=args.topic_base)
session_factory.add(sessiondb_component)
session_factory.add(authorization_session)
log.msg("session_factory")
db_session = DB(component_config,
engine=args.engine,
topic_base=args.topic_base + '.db',
dsn=args.dsn,
debug=args.verbose)
session_factory.add(db_session)
session_factory.userdb.set_session(db_session)
session_factory.sessiondb.set_session(db_session)
## create a WAMP-over-WebSocket transport server factory
##
from autobahn.twisted.websocket import WampWebSocketServerFactory
transport_factory = WampWebSocketServerFactory(session_factory,
debug=args.debug)
transport_factory.setProtocolOptions(failByDrop=False)
## start the server from an endpoint
##
## this address clash detection was a goody I got from stackoverflow:
## http://stackoverflow.com/questions/12007316/exiting-twisted-application-after-listenfailure
server = serverFromString(reactor, args.endpoint)
def listen():
srv = server.listen(transport_factory)
def ListenFailed(reason):
log.msg("On Startup Listen Failed with {}".format(reason))
reactor.stop()
srv.addErrback(ListenFailed)
def addsession():
log.msg("here are three sessions {} {} {}".format(
authorization_session, sessiondb_component, db_session))
qv = {
"sessiondb_component": sessiondb_component._session_id,
"db_session": db_session._session_id,
"authorization_session": authorization_session._session_id
}
session_factory.sessiondb.set_system_sessions(qv)
session_factory.sessiondb.add(0, sessiondb_component._session_id,
sessiondb_component)
session_factory.sessiondb.add(0, db_session._session_id, db_session)
session_factory.sessiondb.add(0, authorization_session._session_id,
authorization_session)
reactor.callWhenRunning(listen)
reactor.callWhenRunning(addsession)
# if we set this router to be mastered by another, then start communicating with it
if args.xsocket is not None:
xdb = Xomponent(config=xomponent_config,
authinfo=ai,
topic_base=args.topic_base,
debug=args.verbose,
command='session',
action='list',
action_args={})
xdb.set_session(db_session)
runner = ApplicationRunner(args.xsocket, args.realm)
runner.run(lambda _: xdb, start_reactor=False)
reactor.run()
def main(pm):
from sputnik.webserver.router.twisted.wamp import RouterFactory
router_factory = RouterFactory()
router_factory.router = SputnikRouter
router_factory.authz_plugins = \
pm.services.get("sputnik.webserver.plugins.authz", [])
router_factory.schema_plugins = \
pm.services.get("sputnik.webserver.plugins.schema", [])
from sputnik.webserver.router.twisted.wamp import RouterSessionFactory
session_factory = RouterSessionFactory(router_factory)
session_factory.session = SputnikRouterSession
authn_stack = [("ip.IPFilter", "requisite"),
("anonymous.AnonymousLogin", "sufficient"),
("cookie.CookieLogin", "sufficient"),
("wampcra.WAMPCRALogin", "requisite"),
("totp.TOTPVerification", "requisite")]
session_factory.plugins = []
for plugin_name, flag in authn_stack:
path = "sputnik.webserver.plugins.authn." + plugin_name
session_factory.plugins.append((pm.plugins[path], flag))
rpc_plugins = pm.services.get("sputnik.webserver.plugins.rpc", [])
feeds_plugins = pm.services.get("sputnik.webserver.plugins.feeds", [])
svc_plugins = rpc_plugins + feeds_plugins
for plugin in svc_plugins:
component_session = plugin
component_session.config.realm = u"sputnik"
session_factory.add(component_session,
plugin.plugin_path.decode("ascii"), u"trusted")
uri = "ws://"
if config.getboolean("webserver", "ssl"):
uri = "wss://"
address = config.get("webserver", "ws_address")
port = config.getint("webserver", "ws_port")
uri += "%s:%s/" % (address, port)
from autobahn.twisted.websocket import WampWebSocketServerFactory
transport_factory = WampWebSocketServerFactory(session_factory,
uri, debug = False, debug_wamp = False)
transport_factory.setProtocolOptions(failByDrop = False)
watchdog(config.get("watchdog", "webserver"))
from twisted.web.server import Site
from autobahn.twisted.resource import WebSocketResource
root = Root()
ws_resource = WebSocketResource(transport_factory)
rest_resource = pm.plugins['sputnik.webserver.rest.RESTProxy']
root.putChild("ws", ws_resource)
root.putChild("api", rest_resource)
site = Site(root)
site.noisy = False
site.log = lambda _: None
from twisted.internet.endpoints import serverFromString, quoteStringArgument
if config.getboolean("webserver", "ssl"):
key = config.get("webserver", "ssl_key")
cert = config.get("webserver", "ssl_cert")
cert_chain = config.get("webserver", "ssl_cert_chain")
# TODO: Add dhparameters
# See https://twistedmatrix.com/documents/14.0.0/core/howto/endpoints.html
server = serverFromString(reactor, b"ssl:%d:privateKey=%s:certKey=%s:extraCertChain=%s:sslmethod=TLSv1_METHOD"
% (port,
quoteStringArgument(key),
quoteStringArgument(cert),
quoteStringArgument(cert_chain)))
else:
server = serverFromString(reactor, b"tcp:%d" % port)
server.listen(site)
def __init__(self, factory, cbdir, config, templates):
"""
Ctor.
:param factory: WAMP session factory.
:type factory: An instance of ..
:param cbdir: The Crossbar.io node directory.
:type cbdir: str
:param config: Crossbar transport configuration.
:type config: dict
"""
self.debug = config.get('debug', False)
self.debug_traffic = config.get('debug_traffic', False)
options = config.get('options', {})
server = "Crossbar/{}".format(crossbar.__version__)
externalPort = options.get('external_port', None)
# explicit list of WAMP serializers
#
if 'serializers' in config:
serializers = []
sers = set(config['serializers'])
if 'json' in sers:
# try JSON WAMP serializer
try:
from autobahn.wamp.serializer import JsonSerializer
serializers.append(JsonSerializer())
except ImportError:
print("Warning: could not load WAMP-JSON serializer")
else:
sers.discard('json')
if 'msgpack' in sers:
# try MsgPack WAMP serializer
try:
from autobahn.wamp.serializer import MsgPackSerializer
serializers.append(MsgPackSerializer())
except ImportError:
print("Warning: could not load WAMP-MsgPack serializer")
else:
sers.discard('msgpack')
if not serializers:
raise Exception("no valid WAMP serializers specified")
if len(sers) > 0:
raise Exception("invalid WAMP serializers specified: {}".format(sers))
else:
serializers = None
WampWebSocketServerFactory.__init__(self,
factory,
serializers=serializers,
url=config.get('url', None),
server=server,
externalPort=externalPort,
debug=self.debug,
debug_wamp=self.debug)
# Crossbar.io node directory
self._cbdir = cbdir
# transport configuration
self._config = config
# Jinja2 templates for 404 etc
self._templates = templates
# cookie tracking
if 'cookie' in config:
if 'database' in config['cookie']:
dbfile = os.path.abspath(os.path.join(self._cbdir, config['cookie']['database']))
self._cookiestore = PersistentCookieStore(dbfile, config['cookie'])
log.msg("Persistent cookie store active: {}".format(dbfile))
else:
self._cookiestore = CookieStore(config['cookie'])
log.msg("Transient cookie store active.")
else:
self._cookiestore = None
# set WebSocket options
set_websocket_options(self, options)
def _run_command_exec_worker(options, reactor=None, personality=None):
"""
Entry point into (native) worker processes. This wires up stuff such that
a worker instance is talking WAMP-over-stdio to the node controller.
"""
import os
import sys
import platform
import signal
# https://coverage.readthedocs.io/en/coverage-4.4.2/subprocess.html#measuring-sub-processes
MEASURING_COVERAGE = False
if 'COVERAGE_PROCESS_START' in os.environ:
try:
import coverage
except ImportError:
pass
else:
# The following will read the environment variable COVERAGE_PROCESS_START,
# and that should be set to the .coveragerc file:
#
# export COVERAGE_PROCESS_START=${PWD}/.coveragerc
#
coverage.process_startup()
MEASURING_COVERAGE = True
# we use an Autobahn utility to import the "best" available Twisted reactor
from autobahn.twisted.choosereactor import install_reactor
reactor = install_reactor(options.reactor)
# make sure logging to something else than stdio is setup _first_
from crossbar._logging import make_JSON_observer, cb_logging_aware
from txaio import make_logger, start_logging
from twisted.logger import globalLogPublisher
from twisted.python.reflect import qual
log = make_logger()
# Print a magic phrase that tells the capturing logger that it supports
# Crossbar's rich logging
print(cb_logging_aware, file=sys.__stderr__)
sys.__stderr__.flush()
flo = make_JSON_observer(sys.__stderr__)
globalLogPublisher.addObserver(flo)
term_print('CROSSBAR[{}]:WORKER_STARTING'.format(options.worker))
# Ignore SIGINT so we get consistent behavior on control-C versus
# sending SIGINT to the controller process. When the controller is
# shutting down, it sends TERM to all its children but ctrl-C
# handling will send a SIGINT to all the processes in the group
# (so then the controller sends a TERM but the child already or
# will very shortly get a SIGINT as well). Twisted installs signal
# handlers, but not for SIGINT if there's already a custom one
# present.
def ignore(sig, frame):
log.debug("Ignoring SIGINT in worker.")
signal.signal(signal.SIGINT, ignore)
# actually begin logging
start_logging(None, options.loglevel)
# get personality klass, eg "crossbar.personality.Personality"
l = options.personality.split('.')
personality_module, personality_klass = '.'.join(l[:-1]), l[-1]
# now load the personality module and class
_mod = importlib.import_module(personality_module)
Personality = getattr(_mod, personality_klass)
# get worker klass, eg "crossbar.worker.container.ContainerController"
l = options.klass.split('.')
worker_module, worker_klass = '.'.join(l[:-1]), l[-1]
# now load the worker module and class
_mod = importlib.import_module(worker_module)
klass = getattr(_mod, worker_klass)
log.info(
'Starting {worker_type}-worker "{worker_id}" on node "{node_id}" (personality "{personality}") and local node management realm "{realm}" .. {worker_class}',
worker_type=hl(klass.WORKER_TYPE),
worker_id=hlid(options.worker),
node_id=hlid(options.node),
realm=hlid(options.realm),
personality=hl(Personality.NAME),
worker_class=hltype(klass),
)
log.info(
'Running as PID {pid} on {python}-{reactor}',
pid=os.getpid(),
python=platform.python_implementation(),
reactor=qual(reactor.__class__).split('.')[-1],
)
if MEASURING_COVERAGE:
log.info(hl(
'Code coverage measurements enabled (coverage={coverage_version}).',
color='green',
bold=True),
coverage_version=coverage.__version__)
# set process title if requested to
#
try:
import setproctitle
except ImportError:
log.debug(
"Could not set worker process title (setproctitle not installed)")
else:
if options.title:
setproctitle.setproctitle(options.title)
else:
setproctitle.setproctitle('crossbar-worker [{}]'.format(
options.klass))
# node directory
#
options.cbdir = os.path.abspath(options.cbdir)
os.chdir(options.cbdir)
# log.msg("Starting from node directory {}".format(options.cbdir))
# set process title if requested to
#
try:
import setproctitle
except ImportError:
log.debug(
"Could not set worker process title (setproctitle not installed)")
else:
if options.title:
setproctitle.setproctitle(options.title)
else:
setproctitle.setproctitle('crossbar-worker [{}]'.format(
options.klass))
from twisted.internet.error import ConnectionDone
from autobahn.twisted.websocket import WampWebSocketServerProtocol
class WorkerServerProtocol(WampWebSocketServerProtocol):
def connectionLost(self, reason):
# the behavior here differs slightly whether we're shutting down orderly
# or shutting down because of "issues"
if isinstance(reason.value, ConnectionDone):
was_clean = True
else:
was_clean = False
try:
# this log message is unlikely to reach the controller (unless
# only stdin/stdout pipes were lost, but not stderr)
if was_clean:
log.info("Connection to node controller closed cleanly")
else:
log.warn("Connection to node controller lost: {reason}",
reason=reason)
# give the WAMP transport a change to do it's thing
WampWebSocketServerProtocol.connectionLost(self, reason)
except:
# we're in the process of shutting down .. so ignore ..
pass
finally:
# after the connection to the node controller is gone,
# the worker is "orphane", and should exit
# determine process exit code
if was_clean:
exit_code = 0
else:
exit_code = 1
# exit the whole worker process when the reactor has stopped
reactor.addSystemEventTrigger('after', 'shutdown', os._exit,
exit_code)
# stop the reactor
try:
reactor.stop()
except ReactorNotRunning:
pass
# if vmprof global profiling is enabled via command line option, this will carry
# the file where vmprof writes its profile data
if _HAS_VMPROF:
_vm_prof = {
# need to put this into a dict, since FDs are ints, and python closures can't
# write to this otherwise
'outfd': None
}
# https://twistedmatrix.com/documents/current/api/twisted.internet.interfaces.IReactorCore.html
# Each "system event" in Twisted, such as 'startup', 'shutdown', and 'persist', has 3 phases:
# 'before', 'during', and 'after' (in that order, of course). These events will be fired
# internally by the Reactor.
def before_reactor_started():
term_print('CROSSBAR[{}]:REACTOR_STARTING'.format(options.worker))
def after_reactor_started():
term_print('CROSSBAR[{}]:REACTOR_STARTED'.format(options.worker))
if _HAS_VMPROF and options.vmprof:
outfn = os.path.join(
options.cbdir,
'.vmprof-worker-{}-{}.dat'.format(options.worker, os.getpid()))
_vm_prof['outfd'] = os.open(outfn,
os.O_RDWR | os.O_CREAT | os.O_TRUNC)
vmprof.enable(_vm_prof['outfd'], period=0.01)
term_print('CROSSBAR[{}]:VMPROF_ENABLED:{}'.format(
options.worker, outfn))
def before_reactor_stopped():
term_print('CROSSBAR[{}]:REACTOR_STOPPING'.format(options.worker))
if _HAS_VMPROF and options.vmprof and _vm_prof['outfd']:
vmprof.disable()
term_print('CROSSBAR[{}]:VMPROF_DISABLED'.format(options.worker))
def after_reactor_stopped():
# FIXME: we are indeed reaching this line, however,
# the log output does not work (it also doesnt work using
# plain old print). Dunno why.
# my theory about this issue is: by the time this line
# is reached, Twisted has already closed the stdout/stderr
# pipes. hence we do an evil trick: we directly write to
# the process' controlling terminal
# https://unix.stackexchange.com/a/91716/52500
term_print('CROSSBAR[{}]:REACTOR_STOPPED'.format(options.worker))
reactor.addSystemEventTrigger('before', 'startup', before_reactor_started)
reactor.addSystemEventTrigger('after', 'startup', after_reactor_started)
reactor.addSystemEventTrigger('before', 'shutdown', before_reactor_stopped)
reactor.addSystemEventTrigger('after', 'shutdown', after_reactor_stopped)
try:
# define a WAMP application session factory
#
from autobahn.wamp.types import ComponentConfig
def make_session():
session_config = ComponentConfig(realm=options.realm,
extra=options)
session = klass(config=session_config,
reactor=reactor,
personality=Personality)
return session
# create a WAMP-over-WebSocket transport server factory
#
from autobahn.twisted.websocket import WampWebSocketServerFactory
transport_factory = WampWebSocketServerFactory(make_session,
'ws://localhost')
transport_factory.protocol = WorkerServerProtocol
transport_factory.setProtocolOptions(failByDrop=False)
# create a protocol instance and wire up to stdio
#
from twisted.python.runtime import platform as _platform
from twisted.internet import stdio
proto = transport_factory.buildProtocol(None)
if _platform.isWindows():
stdio.StandardIO(proto)
else:
stdio.StandardIO(proto, stdout=3)
# now start reactor loop
#
log.info(hl('Entering event reactor ...', color='green', bold=True))
reactor.run()
except Exception as e:
log.info("Unhandled exception: {e}", e=e)
if reactor.running:
reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1)
reactor.stop()
else:
sys.exit(1)
def run(self, make, start_reactor = True):
"""
Run the application component.
:param make: A factory that produces instances of :class:`autobahn.asyncio.wamp.ApplicationSession`
when called with an instance of :class:`autobahn.wamp.types.ComponentConfig`.
:type make: callable
"""
from twisted.internet import reactor
isSecure, host, port, resource, path, params = parseWsUrl(self.url)
## start logging to console
if self.debug or self.debug_wamp or self.debug_app:
log.startLogging(sys.stdout)
## run an embedded router if ask to start standalone
if self.standalone:
from twisted.internet.endpoints import serverFromString
router_factory = RouterFactory()
session_factory = RouterSessionFactory(router_factory)
transport_factory = WampWebSocketServerFactory(session_factory, debug = self.debug, debug_wamp = self.debug_wamp)
transport_factory.setProtocolOptions(failByDrop = False)
server = serverFromString(reactor, "tcp:{0}".format(port))
server.listen(transport_factory)
## factory for use ApplicationSession
def create():
cfg = ComponentConfig(self.realm, self.extra)
try:
session = make(cfg)
except Exception:
## the app component could not be created .. fatal
log.err()
reactor.stop()
else:
session.debug_app = self.debug_app
return session
## create a WAMP-over-WebSocket transport client factory
transport_factory = WampWebSocketClientFactory(create, url = self.url, serializers = self.serializers,
debug = self.debug, debug_wamp = self.debug_wamp)
## start the client from a Twisted endpoint
from twisted.internet.endpoints import clientFromString
if isSecure:
endpoint_descriptor = "ssl:{0}:{1}".format(host, port)
else:
endpoint_descriptor = "tcp:{0}:{1}".format(host, port)
client = clientFromString(reactor, endpoint_descriptor)
client.connect(transport_factory)
## now enter the Twisted reactor loop
if start_reactor:
reactor.run()
##
from autobahn.twisted.choosereactor import install_reactor
reactor = install_reactor()
print("Running on reactor {}".format(reactor))
# create a WAMP router factory
##
from autobahn.twisted.wamp import RouterFactory
router_factory = RouterFactory()
router_factory.router = MyRouter
# create a WAMP router session factory
##
from autobahn.twisted.wamp import RouterSessionFactory
session_factory = RouterSessionFactory(router_factory)
# create a WAMP-over-WebSocket transport server factory
##
from autobahn.twisted.websocket import WampWebSocketServerFactory
transport_factory = WampWebSocketServerFactory(session_factory, debug=args.debug)
transport_factory.setProtocolOptions(failByDrop=False)
# start the server from an endpoint
##
server = serverFromString(reactor, args.endpoint)
server.listen(transport_factory)
# now enter the Twisted reactor loop
##
reactor.run()
def run():
"""
Entry point into (native) worker processes. This wires up stuff such that
a worker instance is talking WAMP-over-stdio to the node controller.
"""
import os
import sys
import platform
import signal
# Ignore SIGINT so we get consistent behavior on control-C versus
# sending SIGINT to the controller process. When the controller is
# shutting down, it sends TERM to all its children but ctrl-C
# handling will send a SIGINT to all the processes in the group
# (so then the controller sends a TERM but the child already or
# will very shortly get a SIGINT as well). Twisted installs signal
# handlers, but not for SIGINT if there's already a custom one
# present.
def ignore(sig, frame):
log.msg("Ignoring SIGINT in worker.")
signal.signal(signal.SIGINT, ignore)
# create the top-level parser
#
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('-d',
'--debug',
action='store_true',
help='Debug on (optional).')
parser.add_argument('--reactor',
default=None,
choices=['select', 'poll', 'epoll', 'kqueue', 'iocp'],
help='Explicit Twisted reactor selection (optional).')
parser.add_argument('-c',
'--cbdir',
type=str,
help="Crossbar.io node directory (required).")
parser.add_argument('-n',
'--node',
type=str,
help='Crossbar.io node ID (required).')
parser.add_argument('-w',
'--worker',
type=str,
help='Crossbar.io worker ID (required).')
parser.add_argument('-r',
'--realm',
type=str,
help='Crossbar.io node (management) realm (required).')
parser.add_argument('-t',
'--type',
choices=['router', 'container'],
help='Worker type (required).')
parser.add_argument('--title',
type=str,
default=None,
help='Worker process title to set (optional).')
options = parser.parse_args()
# make sure logging to something else than stdio is setup _first_
#
from twisted.python import log
from crossbar.twisted.processutil import BareFormatFileLogObserver
flo = BareFormatFileLogObserver(sys.stderr)
log.startLoggingWithObserver(flo.emit)
try:
import setproctitle
except ImportError:
log.msg(
"Warning: could not set worker process title (setproctitle not installed)"
)
else:
# set process title if requested to
#
if options.title:
setproctitle.setproctitle(options.title)
else:
WORKER_TYPE_TO_TITLE = {
'router': 'crossbar-worker [router]',
'container': 'crossbar-worker [container]'
}
setproctitle.setproctitle(
WORKER_TYPE_TO_TITLE[options.type].strip())
# we use an Autobahn utility to import the "best" available Twisted reactor
#
from autobahn.twisted.choosereactor import install_reactor
reactor = install_reactor(options.reactor)
from twisted.python.reflect import qual
log.msg("Running under {} using {} reactor".format(
platform.python_implementation(),
qual(reactor.__class__).split('.')[-1]))
options.cbdir = os.path.abspath(options.cbdir)
os.chdir(options.cbdir)
# log.msg("Starting from node directory {}".format(options.cbdir))
from crossbar.worker.router import RouterWorkerSession
from crossbar.worker.container import ContainerWorkerSession
WORKER_TYPE_TO_CLASS = {
'router': RouterWorkerSession,
'container': ContainerWorkerSession
}
from autobahn.twisted.websocket import WampWebSocketServerProtocol
class WorkerServerProtocol(WampWebSocketServerProtocol):
def connectionLost(self, reason):
try:
# this log message is unlikely to reach the controller (unless
# only stdin/stdout pipes were lost, but not stderr)
log.msg("Connection to node controller lost.")
WampWebSocketServerProtocol.connectionLost(self, reason)
except:
pass
finally:
# loosing the connection to the node controller is fatal:
# stop the reactor and exit with error
if reactor.running:
reactor.addSystemEventTrigger('after', 'shutdown',
os._exit, 1)
reactor.stop()
# if the reactor *isn't* running, we're already shutting down
try:
# create a WAMP application session factory
#
from autobahn.twisted.wamp import ApplicationSessionFactory
from autobahn.wamp.types import ComponentConfig
session_config = ComponentConfig(realm=options.realm, extra=options)
session_factory = ApplicationSessionFactory(session_config)
session_factory.session = WORKER_TYPE_TO_CLASS[options.type]
# create a WAMP-over-WebSocket transport server factory
#
from autobahn.twisted.websocket import WampWebSocketServerFactory
transport_factory = WampWebSocketServerFactory(session_factory,
"ws://localhost",
debug=False,
debug_wamp=False)
transport_factory.protocol = WorkerServerProtocol
transport_factory.setProtocolOptions(failByDrop=False)
# create a protocol instance and wire up to stdio
#
from twisted.internet import stdio
proto = transport_factory.buildProtocol(None)
stdio.StandardIO(proto)
# now start reactor loop
#
log.msg("Entering event loop ..")
reactor.run()
except Exception as e:
log.msg("Unhandled exception: {}".format(e))
if reactor.running:
reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1)
reactor.stop()
else:
sys.exit(1)
def test_minimal(self):
embedded_components, client_components = [], [
Case2_Backend, Case2_Frontend
]
## create a WAMP router factory
##
router_factory = RouterFactory()
## create a WAMP router session factory
##
session_factory = RouterSessionFactory(router_factory)
## .. and create and add an WAMP application session to
## run next to the router
##
config = types.ComponentConfig(realm=self.realm,
extra={'caselog': 'case1.log'})
try:
log = io.open('caselog.log', 'w')
except Exception as e:
print(e)
return
# log = io.open(config.extra['caselog'], 'w')
config.log = log
config.dlog = []
config.components = []
config.all_done = []
for C in embedded_components:
one_done = Deferred()
config.all_done.append(one_done)
c = C(config, one_done)
config.components.append(c)
session_factory.add(c)
if self.transport == "websocket":
## create a WAMP-over-WebSocket transport server factory
##
transport_factory = WampWebSocketServerFactory(
session_factory, debug_wamp=self.debug)
transport_factory.setProtocolOptions(failByDrop=False,
openHandshakeTimeout=0,
closeHandshakeTimeout=0)
elif self.transport in ['rawsocket-json', 'rawsocket-msgpack']:
## create a WAMP-over-RawSocket transport server factory
##
if self.transport == 'rawsocket-msgpack':
serializer = MsgPackSerializer()
elif self.transport == 'rawsocket-json':
serializer = JsonSerializer()
else:
raise Exception("should not arrive here")
transport_factory = WampRawSocketServerFactory(
session_factory, serializer, debug=self.debug)
else:
raise Exception("should not arrive here")
## start the server from an endpoint
##
from twisted.internet import reactor
server = serverFromString(reactor, self.server)
d = server.listen(transport_factory)
def onlisten(port):
config.port = port
d.addCallback(onlisten)
clients = []
clients_d = []
for C in client_components:
## create a WAMP application session factory
##
session_factory = ApplicationSessionFactory(config)
one_done = Deferred()
config.all_done.append(one_done)
def make_make(Klass, done):
def make(config):
c = Klass(config, done)
config.components.append(c)
return c
return make
## .. and set the session class on the factory
##
session_factory.session = make_make(C, one_done)
if self.transport == "websocket":
serializers = [JsonSerializer()]
## create a WAMP-over-WebSocket transport client factory
##
transport_factory = WampWebSocketClientFactory(
session_factory,
serializers=serializers,
url=self.url,
debug_wamp=self.debug)
if True:
def maker(Klass):
class TestClientProtocol(
WampWebSocketClientProtocol):
def onOpen(self):
self.txcnt = 0
self.rxcnt = 0
WampWebSocketClientProtocol.onOpen(self)
def sendMessage(self, bytes, isBinary):
self.txcnt += 1
print("> : {:>3} : {:<20} : {}".format(
self.txcnt, Klass.__name__, bytes))
WampWebSocketClientProtocol.sendMessage(
self, bytes, isBinary)
def onMessage(self, bytes, isBinary):
self.rxcnt += 1
print("< : {:>3} : {:<20} : {}".format(
self.rxcnt, Klass.__name__, bytes))
WampWebSocketClientProtocol.onMessage(
self, bytes, isBinary)
return TestClientProtocol
transport_factory.protocol = maker(C)
else:
transport_factory.protocol = WampWebSocketClientProtocol
transport_factory.setProtocolOptions(
failByDrop=False,
openHandshakeTimeout=0,
closeHandshakeTimeout=0)
elif self.transport in ['rawsocket-json', 'rawsocket-msgpack']:
## create a WAMP-over-RawSocket transport client factory
##
if self.transport == 'rawsocket-msgpack':
serializer = MsgPackSerializer()
elif self.transport == 'rawsocket-json':
serializer = JsonSerializer()
else:
raise Exception("should not arrive here")
transport_factory = WampRawSocketClientFactory(
session_factory, serializer, debug=self.debug)
## start the client from an endpoint
##
cl = clientFromString(reactor, self.client)
clients_d.append(cl.connect(transport_factory))
clients.append(cl)
config.connected_clients = None
def client_connected(res):
config.connected_clients = [
proto for success, proto in res if success
]
DeferredList(clients_d).addCallback(client_connected)
d = DeferredList(config.all_done, consumeErrors=True)
#d = config.components[1]._done
def done(res):
log.flush()
log.close()
if config.port:
config.port.stopListening()
if config.connected_clients:
for proto in config.connected_clients:
proto.transport.abortConnection()
print("Log length: {}".format(len(config.dlog)))
print(config.dlog)
#from twisted.internet import reactor
#reactor.callLater(1, reactor.stop)
def error(err):
print(err)
d.addCallbacks(done, error)
# d2 = Deferred()
return d
def __init__(self, factory, config, templates):
"""
Ctor.
:param factory: WAMP session factory.
:type factory: An instance of ..
:param config: Crossbar transport configuration.
:type config: dict
"""
options = config.get('options', {})
server = "Crossbar/{}".format(crossbar.__version__)
externalPort = options.get('external_port', None)
## explicit list of WAMP serializers
##
if 'serializers' in options:
serializers = []
sers = set(options['serializers'])
if 'json' in sers:
## try JSON WAMP serializer
try:
from autobahn.wamp.serializer import JsonSerializer
serializers.append(JsonSerializer())
except ImportError:
print("Warning: could not load WAMP-JSON serializer")
else:
sers.discard('json')
if 'msgpack' in sers:
## try MsgPack WAMP serializer
try:
from autobahn.wamp.serializer import MsgPackSerializer
serializers.append(MsgPackSerializer())
except ImportError:
print("Warning: could not load WAMP-MsgPack serializer")
else:
sers.discard('msgpack')
if not serializers:
raise Exception("no valid WAMP serializers specified")
if len(sers) > 0:
raise Exception("invalid WAMP serializers specified: {}".format(sers))
else:
serializers = None
WampWebSocketServerFactory.__init__(self,
factory,
serializers = serializers,
url = config.get('url', None),
server = server,
externalPort = externalPort,
debug = config.get('debug', False))
## transport configuration
self._config = config
## Jinja2 templates for 404 etc
self._templates = templates
## cookie tracking
if 'cookie' in config:
self._cookies = {}
## set WebSocket options
set_websocket_options(self, options)
##
from autobahn.twisted.choosereactor import install_reactor
reactor = install_reactor()
print("Running on reactor {}".format(reactor))
## create a WAMP router factory
##
from autobahn.twisted.wamp import RouterFactory
router_factory = RouterFactory()
## create a WAMP router session factory
##
from autobahn.twisted.wamp import RouterSessionFactory
session_factory = RouterSessionFactory(router_factory)
## create a WAMP-over-WebSocket transport server factory
##
from autobahn.twisted.websocket import WampWebSocketServerFactory
transport_factory = WampWebSocketServerFactory(session_factory,
debug=args.debug)
transport_factory.setProtocolOptions(failByDrop=False)
## start the server from an endpoint
##
server = serverFromString(reactor, args.endpoint)
server.listen(transport_factory)
## now enter the Twisted reactor loop
##
reactor.run()
class Node:
"""
A Crossbar.io node is the running a controller process
and one or multiple worker processes.
A single Crossbar.io node runs exactly one instance of
this class, hence this class can be considered a system
singleton.
"""
def __init__(self, reactor, options):
"""
Ctor.
:param reactor: Reactor to run on.
:type reactor: obj
:param cbdir: Crossbar.io node directory to run from.
:type cbdir: str
"""
self.debug = options.debug
self._cbdir = options.cbdir
self._reactor = reactor
self._worker_processes = {}
## node name: FIXME
self._node_name = "{}-{}".format(socket.getfqdn(), os.getpid())
self._node_name.replace('-', '_')
self._node_name = '918234'
self._node_realm = 'crossbar'
self._node_controller_session = None
## node management
self._management_url = "ws://127.0.0.1:7000"
#self._management_url = "wss://cloud.crossbar.io"
self._management_realm = "crossbar.cloud.aliceblue"
@inlineCallbacks
def start(self):
"""
Starts this node. This will start a node controller
and then spawn new worker processes as needed.
The node controller will watch spawned processes,
communicate via stdio with the worker, and start
and restart the worker processes as needed.
"""
try:
import setproctitle
except ImportError:
log.msg("Warning, could not set process title (setproctitle not installed)")
else:
setproctitle.setproctitle("Crossbar.io Node Controller")
## the node controller singleton WAMP application session
##
self._node_controller_session = NodeControllerSession(self)
## router and factory that creates router sessions
##
self._router_factory = RouterFactory(
options = types.RouterOptions(uri_check = types.RouterOptions.URI_CHECK_LOOSE),
debug = False)
self._router_session_factory = RouterSessionFactory(self._router_factory)
## add the node controller singleton session to the router
##
self._router_session_factory.add(self._node_controller_session)
## Detect WAMPlets
##
for wpl in self._node_controller_session.list_wamplets():
log.msg("WAMPlet detected: {}.{}".format(wpl['dist'], wpl['name']))
yield self.start_from_local_config(configfile = os.path.join(self._cbdir, 'config.json'))
self.start_local_management_transport(endpoint_descriptor = "tcp:9000")
def start_remote_management_client(self):
from crossbar.management import NodeManagementSession
management_session_factory = ApplicationSessionFactory()
management_session_factory.session = NodeManagementSession
management_session_factory.node_controller_session = node_controller_session
management_transport_factory = WampWebSocketClientFactory(management_session_factory, "ws://127.0.0.1:7000")
management_transport_factory.setProtocolOptions(failByDrop = False)
management_client = clientFromString(self._reactor, "tcp:127.0.0.1:7000")
management_client.connect(management_transport_factory)
def start_local_management_transport(self, endpoint_descriptor):
## create a WAMP-over-WebSocket transport server factory
##
from autobahn.twisted.websocket import WampWebSocketServerFactory
from twisted.internet.endpoints import serverFromString
self._router_server_transport_factory = WampWebSocketServerFactory(self._router_session_factory, debug = self.debug)
self._router_server_transport_factory.setProtocolOptions(failByDrop = False)
## start the WebSocket server from an endpoint
##
self._router_server = serverFromString(self._reactor, endpoint_descriptor)
self._router_server.listen(self._router_server_transport_factory)
@inlineCallbacks
def start_from_local_config(self, configfile):
## load Crossbar.io node configuration
##
configfile = os.path.abspath(configfile)
log.msg("Loading from local config '{}' ..".format(configfile))
config = check_config_file(configfile, silence = True)
## startup the node from configuration file
##
yield self._node_controller_session.run_node_config(config)
log.msg("Local configuration loaded.")
