def __init__(self, options, session):
"""
Ctor.
:param options: Options for path service from configuration.
:type options: dict
:param session: Instance of `ApplicationSession` to be used for forwarding events.
:type session: obj
"""
Resource.__init__(self)
self._options = options
self._session = session
self.log = make_logger()
self._key = None
if 'key' in options:
self._key = options['key'].encode('utf8')
self._secret = None
if 'secret' in options:
self._secret = options['secret'].encode('utf8')
self._post_body_limit = int(options.get('post_body_limit', 0))
self._timestamp_delta_limit = int(options.get('timestamp_delta_limit', 300))
self._require_ip = None
if 'require_ip' in options:
self._require_ip = [IPNetwork(net) for net in options['require_ip']]
self._require_tls = options.get('require_tls', None)
def test_start_router_component_invalid_type(self):
"""
Trying to start a component with an invalid type fails.
"""
log_list = []
r = router.RouterWorkerSession(config=self.config, reactor=reactor)
r.log = make_logger(observer=log_list.append, log_level="debug")
# Open the transport
transport = FakeWAMPTransport(r)
r.onOpen(transport)
realm_config = {
u"name": u"realm1",
u'roles': []
}
r.start_router_realm("realm1", realm_config)
component_config = {
"type": u"notathingcrossbarsupports",
"realm": u"realm1"
}
with self.assertRaises(ApplicationError) as e:
r.start_router_component("newcomponent", component_config)
self.assertIn(
("ERROR: invalid router component configuration (invalid value "
"'notathingcrossbarsupports' for component type)"),
str(e.exception.args[0]))
self.assertEqual(len(r.get_router_components()), 0)
def __init__(self, reactor, options):
"""
Ctor.
:param reactor: Reactor to run on.
:type reactor: obj
:param options: Options from command line.
:type options: obj
"""
self.log = make_logger()
self.options = options
# the reactor under which we run
self._reactor = reactor
# shortname for reactor to run (when given via explicit option) or None
self._reactor_shortname = options.reactor
# node directory
self._cbdir = options.cbdir
# the node's name (must be unique within the management realm)
self._node_id = None
# the node's management realm
self._realm = None
# node controller session (a singleton ApplicationSession embedded
# in the node's management router)
self._controller = None
def _cleanup_worker(reactor, worker):
"""
This is called during reactor shutdown and ensures we wait for our
subprocesses to shut down nicely.
"""
log = make_logger()
try:
log.info("sending TERM to subprocess {pid}", pid=worker.pid)
worker.proto.transport.signalProcess('TERM')
# wait for the subprocess to shutdown; could add a timeout
# after which we send a KILL maybe?
d = Deferred()
def protocol_closed(_):
log.debug("{pid} exited", pid=worker.pid)
d.callback(None)
# await worker's timely demise
worker.exit.addCallback(protocol_closed)
def timeout(tried):
if d.called:
return
log.info("waiting for {pid} to exit...", pid=worker.pid)
reactor.callLater(1, timeout, tried + 1)
if tried > 20: # or just wait forever?
log.info("Sending SIGKILL to {pid}", pid=worker.pid)
worker.proto.transport.signalProcess('KILL')
d.callback(None) # or recurse more?
timeout(0)
return d
except ProcessExitedAlready:
pass # ignore; it's already dead
def _appsession_loader(config):
"""
Load a class or a WAMPlet from C{config}.
"""
log = make_logger()
if config['type'] == 'class':
try:
klassname = config['classname']
log.debug("Starting class '{klass}'", klass=klassname)
c = klassname.split('.')
module_name, klass_name = '.'.join(c[:-1]), c[-1]
module = importlib.import_module(module_name)
component = getattr(module, klass_name)
if not issubclass(component, ApplicationSession):
raise ApplicationError(
u"crossbar.error.class_import_failed", "session not derived of ApplicationSession"
)
except Exception as e:
emsg = "Failed to import class '{}'\n{}".format(
klassname, Failure(e).getTraceback())
log.debug(emsg)
log.debug("PYTHONPATH: {pythonpath}", pythonpath=sys.path)
raise ApplicationError(
u"crossbar.error.class_import_failed",
emsg,
pythonpath=sys.path
)
elif config['type'] == 'wamplet':
try:
dist = config['package']
name = config['entrypoint']
log.debug("Starting WAMPlet '{}/{}'".format(dist, name))
# component is supposed to make instances of ApplicationSession
component = pkg_resources.load_entry_point(
dist, 'autobahn.twisted.wamplet', name)
except Exception as e:
emsg = "Failed to import wamplet '{}/{}'\n{}".format(
dist, name, Failure(e).getTraceback())
log.error(emsg)
raise ApplicationError(u"crossbar.error.class_import_failed", emsg)
else:
raise ApplicationError(
u"crossbar.error.invalid_configuration",
"invalid component type '{}'".format(config['type'])
)
return component
def test_set_global_does_not_change_explicit_loggers(self):
"""
Setting the global log level does not change loggers that have an
explicit level set.
"""
log = make_logger("info")
self.assertEqual(log._log_level, "info")
_logging.set_global_log_level("warn")
self.assertEqual(log._log_level, "info")
def test_set_global_changes_loggers(self):
"""
Setting the global log level changes the level of all loggers that were
not instantiated with a level.
"""
log = make_logger()
self.assertEqual(log._log_level, "info")
_logging.set_global_log_level("warn")
self.assertEqual(log._log_level, "warn")
def test_basic(self):
stream = NativeStringIO()
observer = _logging.make_stdout_observer(_file=stream)
log = make_logger(observer=observer)
log.info("Hi!", log_system="foo")
result = stream.getvalue()
self.assertIn(u"[foo]", result)
def test_logger_failure(self):
"""
The failure method catches the in-flight exception.
"""
log = make_logger("info", logger=Mock)
try:
1 / 0
except:
log.failure("Failure happened!")
log.logger.failure.assert_called_once()
def test_logger_failure_not_called(self):
"""
The failure method isn't called under 'none'.
"""
log = make_logger("none", logger=Mock)
try:
1 / 0
except:
log.failure("Failure happened!")
self.assertEqual(log.logger.failure.call_count, 0)
def test_logger_failure(self):
"""
The failure method catches the in-flight exception.
"""
log = make_logger("info", logger=Mock)
try:
1 / 0
except:
log.failure("Failure happened!")
self.assertEqual(log.logger.failure.call_count, 1)
def test_capturer(self):
"""
The log capturer is a context manager that captures the logs emitted
inside it.
"""
log = _logging.make_logger("info")
with _logging.LogCapturer() as l:
log.info("Whee!", log_category="CB500", foo="bar")
self.assertEqual(len(l.get_category("CB500")), 1)
self.assertEqual(l.get_category("CB500")[0]["foo"], "bar")
def test_logger_emits(self):
"""
A Logger emits messages through to its child logger.
"""
log = make_logger("trace", logger=Mock)
log.error("Foo happened!!!")
log.logger.error.assert_called_with("Foo happened!!!")
log.warn("Stuff", foo="bar")
log.logger.warn.assert_called_with("Stuff", foo="bar")
log.trace("Stuff that's trace", foo="bar")
log.logger.debug.assert_called_with("Stuff that's trace", foo="bar", cb_trace=1)
def __init__(self, controller, id, who, keeplog=None):
"""
Ctor.
:param controller: The node controller this worker was created by.
:type controller: instance of NodeControllerSession
:param id: The ID of the worker.
:type id: str
:param who: Who triggered creation of this worker.
:type who: str
:param keeplog: If not `None`, buffer log message received to be later
retrieved via getlog(). If `0`, keep infinite log internally.
If `> 0`, keep at most such many log entries in buffer.
:type keeplog: int or None
"""
self._logger = make_logger()
self._controller = controller
self.id = id
self.who = who
self.pid = None
self.status = "starting"
self.created = datetime.utcnow()
self.connected = None
self.started = None
self._log_entries = deque(maxlen=10)
if platform.isWindows():
self._log_fds = [2]
else:
self._log_fds = [1, 2]
self._log_lineno = 0
self._log_topic = 'crossbar.node.{}.worker.{}.on_log'.format(self._controller._node_id, self.id)
self._log_rich = None # Does not support rich logs
# track stats for worker->controller traffic
self._stats = {}
self._stats_printer = None
# A deferred that resolves when the worker is ready.
self.ready = Deferred()
# A deferred that resolves when the worker has exited.
self.exit = Deferred()
self.exit.addBoth(self._dump_remaining_log)
def test_basic(self):
"""
We can instantiate a RouterWorkerSession.
"""
log_list = []
r = router.RouterWorkerSession(config=self.config, reactor=reactor)
r.log = make_logger(observer=log_list.append, log_level="debug")
# Open the transport
transport = FakeWAMPTransport(r)
r.onOpen(transport)
# XXX depends on log-text; perhaps a little flaky...
self.assertIn("running as", log_list[-1]["log_format"])
def test_basic(self):
"""
We can instantiate a RouterWorkerSession.
"""
log_list = []
r = router.RouterWorkerSession(config=self.config)
r.log = make_logger(observer=log_list.append, log_level="debug")
# Open the transport
transport = FakeWAMPTransport(r)
r.onOpen(transport)
# Should have 35 registers, all for the management interface
self.assertEqual(len(transport._get(Register)), 35)
self.assertIn("ready", log_list[-1]["log_format"])
def test_basic(self):
"""
We can instantiate a RouterWorkerSession.
"""
log_list = []
r = router.RouterWorkerSession(config=self.config, reactor=reactor)
r.log = make_logger(observer=log_list.append, log_level="debug")
# Open the transport
transport = FakeWAMPTransport(r)
r.onOpen(transport)
# Should have 35 registers, all for the management interface
self.assertEqual(len(transport._get(Register)), 35)
# XXX depends on log-text; perhaps a little flaky...
self.assertIn("running as", log_list[-1]["log_format"])
def test_basic(self):
"""
The JSON observer outputs a stream of log events.
"""
stream = StringIO()
observer = _logging.make_JSON_observer(stream)
log = make_logger(observer=observer)
log.info("Hello")
result = stream.getvalue()
log_entry = json.loads(result[:-1])
self.assertEqual(result[-1], _logging.record_separator)
self.assertEqual(len(log_entry.keys()), 4)
self.assertEqual(log_entry["level"], u"info")
self.assertEqual(log_entry["text"], u"Hello")
def run_command_init(options, **kwargs):
"""
Subcommand "crossbar init".
"""
log = make_logger()
from crossbar.controller.template import Templates
templates = Templates()
if options.template not in templates:
log.info(
"Huh, sorry. There is no template named '{options.template}'. Try 'crossbar templates' to list the templates available.",
options=options,
)
sys.exit(1)
if options.appdir is None:
options.appdir = "."
else:
if os.path.exists(options.appdir):
raise Exception("app directory '{}' already exists".format(options.appdir))
try:
os.mkdir(options.appdir)
except Exception as e:
raise Exception("could not create application directory '{}' ({})".format(options.appdir, e))
else:
log.info("Crossbar.io application directory '{options.appdir}' created", options=options)
options.appdir = os.path.abspath(options.appdir)
log.info("Initializing application template '{options.template}' in directory '{options.appdir}'", options=options)
get_started_hint = templates.init(options.appdir, options.template)
log.info("Application template initialized")
if get_started_hint:
log.info("\n{}\n".format(get_started_hint))
else:
log.info(
"\nTo start your node, run 'crossbar start --cbdir {cbdir}'\n",
cbdir=os.path.abspath(os.path.join(options.appdir, ".crossbar")),
)
def run_command_status(options, **kwargs):
"""
Subcommand "crossbar status".
"""
log = make_logger()
# check if there is a Crossbar.io instance currently running from
# the Crossbar.io node directory at all
#
pid_data = check_is_running(options.cbdir)
if pid_data is None:
# https://docs.python.org/2/library/os.html#os.EX_UNAVAILABLE
# https://www.freebsd.org/cgi/man.cgi?query=sysexits&sektion=3
log.info("No Crossbar.io instance is currently running from node directory {cbdir}.",
cbdir=options.cbdir)
sys.exit(getattr(os, 'EX_UNAVAILABLE', 1))
else:
log.info("A Crossbar.io instance is running from node directory {cbdir} (PID {pid}).",
cbdir=options.cbdir, pid=pid_data['pid'])
sys.exit(0)
def test_logger_emits_if_higher(self):
"""
A Logger that has a log level of a higher severity will not emit
messages of a lower severity.
"""
log = make_logger("info", logger=Mock)
log.error("Error!")
log.debug("Debug!")
log.info("Info!")
log.trace("Trace!")
log.emit(LogLevel.info, "Infoooo!")
self.assertEqual(log.logger.failure.call_count, 0)
self.assertEqual(log.logger.critical.call_count, 0)
self.assertEqual(log.logger.error.call_count, 1)
self.assertEqual(log.logger.warn.call_count, 0)
self.assertEqual(log.logger.info.call_count, 2)
self.assertEqual(log.logger.debug.call_count, 0)
self.assertEqual(log.logger.trace.call_count, 0)
def test_failure(self):
"""
Failures include the stacktrace.
"""
stream = StringIO()
observer = _logging.make_JSON_observer(stream)
log = make_logger(observer=observer)
try:
1 / 0
except:
log.failure("Oh no")
result = stream.getvalue()
log_entry = json.loads(result[:-1])
self.assertEqual(result[-1], _logging.record_separator)
self.assertEqual(len(log_entry.keys()), 4)
self.assertIn(u"ZeroDivisionError", log_entry["text"])
self.assertIn(u"Oh no", log_entry["text"])
self.assertEqual(log_entry["level"], u"critical")
def test_raising_during_encoding(self):
"""
Non-JSON-serialisable parameters are repr()'d.
"""
stream = StringIO()
observer = _logging.make_JSON_observer(stream)
log = make_logger(observer=observer)
class BadThing(object):
def __repr__(self):
raise Exception()
log.info("hi {obj}", obj=BadThing())
result = stream.getvalue()
log_entry = json.loads(result[:-1])
self.assertEqual(result[-1], _logging.record_separator)
self.assertEqual(len(log_entry.keys()), 3)
self.assertIn(u"MESSAGE LOST", log_entry["text"])
self.assertEqual(log_entry["level"], u"error")
def test_failure(self):
"""
Failures include the stacktrace.
"""
stream = StringIO()
observer = _logging.make_JSON_observer(stream)
log = make_logger(observer=observer)
try:
1 / 0
except:
log.failure("Oh no")
result = stream.getvalue()
log_entry = json.loads(result[:-1])
self.assertEqual(result[-1], _logging.record_separator)
self.assertEqual(len(log_entry.keys()), 4)
self.assertIn(u"ZeroDivisionError", log_entry["text"])
self.assertIn(u"Oh no", log_entry["text"])
self.assertEqual(log_entry["level"], u"critical")
class NodeManagementSession(ApplicationSession):
"""
This session is used for any uplink CDC connection.
"""
log = make_logger()
def onConnect(self):
authid = self.config.extra['authid']
realm = self.config.realm
self.log.info("Connected. Joining realm '{}' as '{}' ..".format(
realm, authid))
self.join(realm, [u"wampcra"], authid)
def onChallenge(self, challenge):
if challenge.method == u"wampcra":
authkey = self.config.extra['authkey'].encode('utf8')
signature = auth.compute_wcs(
authkey, challenge.extra['challenge'].encode('utf8'))
return signature.decode('ascii')
else:
raise Exception(
"don't know how to compute challenge for authmethod {}".format(
challenge.method))
def onJoin(self, details):
self.log.info("Joined realm '{realm}' on uplink CDC router",
realm=details.realm)
self.config.extra['onready'].callback(self)
def onLeave(self, details):
if details.reason != u"wamp.close.normal":
self.log.warn("Session detached: {}".format(details))
else:
self.log.debug("Session detached: {}".format(details))
self.disconnect()
def onDisconnect(self):
self.log.debug("Disconnected.")
def test_not_json_serialisable(self):
"""
Non-JSON-serialisable parameters are repr()'d.
"""
stream = StringIO()
observer = _logging.make_JSON_observer(stream)
log = make_logger(observer=observer)
try:
1 / 0
except:
log.failure("Oh no", obj=observer)
result = stream.getvalue()
log_entry = json.loads(result[:-1])
self.assertEqual(result[-1], _logging.record_separator)
self.assertEqual(len(log_entry.keys()), 5)
self.assertIn(u"ZeroDivisionError", log_entry["text"])
self.assertIn(u"Oh no", log_entry["text"])
self.assertIn(u"<function ", log_entry["obj"])
self.assertEqual(log_entry["level"], u"critical")
def test_repr_formatting(self):
"""
Non-JSON-serialisable parameters are repr()'d, and any curly brackets
in the result are escaped.
"""
stream = StringIO()
observer = _logging.make_JSON_observer(stream)
log = make_logger(observer=observer)
class BracketThing(object):
def __repr__(self):
return "<BracketThing kwargs={}>"
log.info("hi {obj}", obj=BracketThing())
result = stream.getvalue()
log_entry = json.loads(result[:-1])
self.assertEqual(result[-1], _logging.record_separator)
self.assertEqual(len(log_entry.keys()), 5)
self.assertEqual(u"hi <BracketThing kwargs={{}}>", log_entry["text"])
self.assertEqual(log_entry["level"], u"info")
def test_repr_formatting(self):
"""
Non-JSON-serialisable parameters are repr()'d, and any curly brackets
in the result are escaped.
"""
stream = StringIO()
observer = _logging.make_JSON_observer(stream)
log = make_logger(observer=observer)
class BracketThing(object):
def __repr__(self):
return "<BracketThing kwargs={}>"
log.info("hi {obj}", obj=BracketThing())
result = stream.getvalue()
log_entry = json.loads(result[:-1])
self.assertEqual(result[-1], _logging.record_separator)
self.assertEqual(len(log_entry.keys()), 5)
self.assertEqual(u"hi <BracketThing kwargs={{}}>", log_entry["text"])
self.assertEqual(log_entry["level"], u"info")
def test_not_json_serialisable(self):
"""
Non-JSON-serialisable parameters are repr()'d.
"""
stream = StringIO()
observer = _logging.make_JSON_observer(stream)
log = make_logger(observer=observer)
try:
1 / 0
except:
log.failure("Oh no", obj=observer)
result = stream.getvalue()
log_entry = json.loads(result[:-1])
self.assertEqual(result[-1], _logging.record_separator)
self.assertEqual(len(log_entry.keys()), 5)
self.assertIn(u"ZeroDivisionError", log_entry["text"])
self.assertIn(u"Oh no", log_entry["text"])
self.assertIn(u"<function ", log_entry["obj"])
self.assertEqual(log_entry["level"], u"critical")
def test_raising_during_encoding(self):
"""
Non-JSON-serialisable parameters are repr()'d, and if that's impossible
then the message is lost.
"""
stream = StringIO()
observer = _logging.make_JSON_observer(stream)
log = make_logger(observer=observer)
class BadThing(object):
def __repr__(self):
raise Exception()
log.info("hi {obj}", obj=BadThing())
result = stream.getvalue()
log_entry = json.loads(result[:-1])
self.assertEqual(result[-1], _logging.record_separator)
self.assertEqual(len(log_entry.keys()), 3)
self.assertIn(u"MESSAGE LOST", log_entry["text"])
self.assertEqual(log_entry["level"], u"error")
def test_start_router_component_fails(self):
"""
Trying to start a class-based router component that gets an error on
importing fails.
"""
log_list = []
r = router.RouterWorkerSession(config=self.config, reactor=reactor)
r.log = make_logger(observer=log_list.append, log_level="debug")
# Open the transport
transport = FakeWAMPTransport(r)
r.onOpen(transport)
realm_config = {
u"name": u"realm1",
u'roles': [{u'name': u'anonymous',
u'permissions': [{u'subscribe': True,
u'register': True, u'call': True,
u'uri': u'*', u'publish': True}]}]
}
r.start_router_realm("realm1", realm_config)
component_config = {
"type": u"class",
"classname": u"thisisathing.thatdoesnot.exist",
"realm": u"realm1"
}
with self.assertRaises(ApplicationError) as e:
r.start_router_component("newcomponent", component_config)
self.assertIn(
"Failed to import class 'thisisathing.thatdoesnot.exist'",
str(e.exception.args[0]))
self.assertEqual(len(r.get_router_components()), 0)
def run_command_status(options, **kwargs):
"""
Subcommand "crossbar status".
"""
log = make_logger()
# check if there is a Crossbar.io instance currently running from
# the Crossbar.io node directory at all
#
pid_data = check_is_running(options.cbdir)
if pid_data is None:
# https://docs.python.org/2/library/os.html#os.EX_UNAVAILABLE
# https://www.freebsd.org/cgi/man.cgi?query=sysexits&sektion=3
log.info(
"No Crossbar.io instance is currently running from node directory {cbdir}.",
cbdir=options.cbdir)
sys.exit(getattr(os, 'EX_UNAVAILABLE', 1))
else:
log.info(
"A Crossbar.io instance is running from node directory {cbdir} (PID {pid}).",
cbdir=options.cbdir,
pid=pid_data['pid'])
sys.exit(0)
def test_unicode_logs(self):
"""
Unicode is JSON serialised correctly.
"""
stream = StringIO()
observer = _logging.make_JSON_observer(stream)
log = make_logger(observer=observer)
try:
if PY3:
raise Exception(u"\u2603")
else:
raise Exception(u"\u2603".encode('utf-8'))
except:
log.failure("Oh no")
result = stream.getvalue()
log_entry = json.loads(result[:-1])
self.assertEqual(result[-1], _logging.record_separator)
self.assertEqual(len(log_entry.keys()), 4)
self.assertIn(u"\u2603", log_entry["text"])
self.assertEqual(log_entry["level"], u"critical")
def test_unicode_logs(self):
"""
Unicode is JSON serialised correctly.
"""
stream = StringIO()
observer = _logging.make_JSON_observer(stream)
log = make_logger(observer=observer)
try:
if PY3:
raise Exception(u"\u2603")
else:
raise Exception(u"\u2603".encode('utf-8'))
except:
log.failure("Oh no")
result = stream.getvalue()
log_entry = json.loads(result[:-1])
self.assertEqual(result[-1], _logging.record_separator)
self.assertEqual(len(log_entry.keys()), 4)
self.assertIn(u"\u2603", log_entry["text"])
self.assertEqual(log_entry["level"], u"critical")
class WebSocketTesteeServerProtocol(WebSocketServerProtocol):
log = make_logger()
def onMessage(self, payload, isBinary):
self.sendMessage(payload, isBinary)
def sendServerStatus(self, redirectUrl=None, redirectAfter=0):
"""
Used to send out server status/version upon receiving a HTTP/GET without
upgrade to WebSocket header (and option serverStatus is True).
"""
try:
page = self.factory._templates.get_template(
'cb_ws_testee_status.html')
self.sendHtml(
page.render(redirectUrl=redirectUrl,
redirectAfter=redirectAfter,
cbVersion=crossbar.__version__,
wsUri=self.factory.url))
except Exception as e:
self.log.warn(
"Error rendering WebSocket status page template: {}".format(e))
def _cleanup_worker(reactor, worker):
"""
This is called during reactor shutdown and ensures we wait for our
subprocesses to shut down nicely.
"""
log = make_logger()
try:
log.info("sending TERM to subprocess {pid}", pid=worker.pid)
worker.proto.transport.signalProcess('TERM')
# wait for the subprocess to shutdown; could add a timeout
# after which we send a KILL maybe?
d = Deferred()
def protocol_closed(_):
log.debug("{pid} exited", pid=worker.pid)
d.callback(None)
# await worker's timely demise
worker.exit.addCallback(protocol_closed)
def timeout(tried):
if d.called:
return
log.info("waiting for {pid} to exit...", pid=worker.pid)
reactor.callLater(1, timeout, tried + 1)
if tried > 20: # or just wait forever?
log.info("Sending SIGKILL to {pid}", pid=worker.pid)
try:
worker.proto.transport.signalProcess('KILL')
except ProcessExitedAlready:
pass # ignore; it's already dead
d.callback(None) # or recurse more?
timeout(0)
return d
except ProcessExitedAlready:
pass # ignore; it's already dead
def run_command_clean_cookies(options, **kwargs):
"""
Subcommand "crossbar clean-cookies".
"""
from crossbar.router.cookiestore import CookieFileCleaner, CookieStoreFileBacked
log = make_logger()
configfile = os.path.join(options.cbdir, options.config)
with open(configfile, 'r') as infile:
config = json.load(infile) # type: dict
for worker in config['workers']:
for transport in worker['transports']: # type: dict
if 'cookie' in transport:
store = transport['cookie']['store']
if store['type'] == 'file':
cookiestore = CookieStoreFileBacked(store['filename'], transport['cookie'])
cleaner = CookieFileCleaner(cookiestore)
cleaner.clean()
log.info('{filename} cookie file cleaned.'.format(filename=store['filename']))
class Broker(object):
"""
Basic WAMP broker.
"""
log = make_logger()
def __init__(self, router, options=None):
"""
:param router: The router this dealer is part of.
:type router: Object that implements :class:`crossbar.router.interfaces.IRouter`.
:param options: Router options.
:type options: Instance of :class:`crossbar.router.types.RouterOptions`.
"""
self._router = router
self._options = options or RouterOptions()
# subscription map managed by this broker
self._subscription_map = UriObservationMap()
# map: session -> set of subscriptions (needed for detach)
self._session_to_subscriptions = {}
# check all topic URIs with strict rules
self._option_uri_strict = self._options.uri_check == RouterOptions.URI_CHECK_STRICT
# supported features from "WAMP Advanced Profile"
self._role_features = role.RoleBrokerFeatures(
publisher_identification=True,
pattern_based_subscription=True,
session_meta_api=True,
subscription_meta_api=True,
subscriber_blackwhite_listing=True,
publisher_exclusion=True,
subscription_revocation=True,
payload_transparency=True,
payload_encryption_cryptobox=True)
# store for event history
if self._router._store:
self._event_store = self._router._store.event_store
else:
self._event_store = None
# if there is a store, let the store attach itself to all the subscriptions
# it is configured to track
if self._event_store:
self._event_store.attach_subscription_map(self._subscription_map)
def attach(self, session):
"""
Implements :func:`crossbar.router.interfaces.IBroker.attach`
"""
if session not in self._session_to_subscriptions:
self._session_to_subscriptions[session] = set()
else:
raise Exception(u"session with ID {} already attached".format(
session._session_id))
def detach(self, session):
"""
Implements :func:`crossbar.router.interfaces.IBroker.detach`
"""
if session in self._session_to_subscriptions:
for subscription in self._session_to_subscriptions[session]:
was_subscribed, was_last_subscriber = self._subscription_map.drop_observer(
session, subscription)
# publish WAMP meta events
#
if self._router._realm:
service_session = self._router._realm.session
if service_session and not subscription.uri.startswith(
u'wamp.'):
if was_subscribed:
service_session.publish(
u'wamp.subscription.on_unsubscribe',
session._session_id, subscription.id)
if was_last_subscriber:
service_session.publish(
u'wamp.subscription.on_delete',
session._session_id, subscription.id)
del self._session_to_subscriptions[session]
else:
raise Exception("session with ID {} not attached".format(
session._session_id))
def processPublish(self, session, publish):
"""
Implements :func:`crossbar.router.interfaces.IBroker.processPublish`
"""
# check topic URI: for PUBLISH, must be valid URI (either strict or loose), and
# all URI components must be non-empty
if self._option_uri_strict:
uri_is_valid = _URI_PAT_STRICT_NON_EMPTY.match(publish.topic)
else:
uri_is_valid = _URI_PAT_LOOSE_NON_EMPTY.match(publish.topic)
if not uri_is_valid:
if publish.acknowledge:
reply = message.Error(
message.Publish.MESSAGE_TYPE, publish.request,
ApplicationError.INVALID_URI, [
u"publish with invalid topic URI '{0}' (URI strict checking {1})"
.format(publish.topic, self._option_uri_strict)
])
self._router.send(session, reply)
return
# disallow publication to topics starting with "wamp." and "crossbar." other than for
# trusted sessions (that are sessions built into Crossbar.io)
#
if session._authrole is not None and session._authrole != u"trusted":
is_restricted = publish.topic.startswith(
u"wamp.") or publish.topic.startswith(u"crossbar.")
if is_restricted:
if publish.acknowledge:
reply = message.Error(
message.Publish.MESSAGE_TYPE, publish.request,
ApplicationError.INVALID_URI, [
u"publish with restricted topic URI '{0}'".format(
publish.topic)
])
self._router.send(session, reply)
return
# get subscriptions active on the topic published to
#
subscriptions = self._subscription_map.match_observations(
publish.topic)
# check if the event is being persisted by checking if we ourself are among the observers
# on _any_ matching subscription
# we've been previously added to observer lists on subscriptions ultimately from
# node configuration and during the broker starts up.
store_event = False
if self._event_store:
for subscription in subscriptions:
if self._event_store in subscription.observers:
store_event = True
break
if store_event:
self.log.debug('Persisting event on topic "{topic}"',
topic=publish.topic)
# go on if (otherwise there isn't anything to do anyway):
#
# - there are any active subscriptions OR
# - the publish is to be acknowledged OR
# - the event is to be persisted
#
if subscriptions or publish.acknowledge or store_event:
# validate payload
#
if publish.payload is None:
try:
self._router.validate('event', publish.topic, publish.args,
publish.kwargs)
except Exception as e:
if publish.acknowledge:
reply = message.Error(
message.Publish.MESSAGE_TYPE, publish.request,
ApplicationError.INVALID_ARGUMENT, [
u"publish to topic URI '{0}' with invalid application payload: {1}"
.format(publish.topic, e)
])
self._router.send(session, reply)
return
# authorize PUBLISH action
#
d = self._router.authorize(session, publish.topic, u'publish')
def on_authorize_success(authorization):
# the call to authorize the action _itself_ succeeded. now go on depending on whether
# the action was actually authorized or not ..
#
if not authorization[u'allow']:
if publish.acknowledge:
reply = message.Error(
message.Publish.MESSAGE_TYPE, publish.request,
ApplicationError.NOT_AUTHORIZED, [
u"session not authorized to publish to topic '{0}'"
.format(publish.topic)
])
self._router.send(session, reply)
else:
# new ID for the publication
#
publication = util.id()
# persist event (this is done only once, regardless of the number of subscriptions
# the event matches on)
#
if store_event:
self._event_store.store_event(session._session_id,
publication,
publish.topic,
publish.args,
publish.kwargs)
# send publish acknowledge immediately when requested
#
if publish.acknowledge:
reply = message.Published(publish.request, publication)
self._router.send(session, reply)
# publisher disclosure
#
if authorization[u'disclose']:
publisher = session._session_id
publisher_authid = session._authid
publisher_authrole = session._authrole
else:
publisher = None
publisher_authid = None
publisher_authrole = None
# skip publisher
#
if publish.exclude_me is None or publish.exclude_me:
me_also = False
else:
me_also = True
# iterate over all subscriptions ..
#
for subscription in subscriptions:
# persist event history, but check if it is persisted on the individual subscription!
#
if store_event and self._event_store in subscription.observers:
self._event_store.store_event_history(
publication, subscription.id)
# initial list of receivers are all subscribers on a subscription ..
#
receivers = subscription.observers
# filter by "eligible" receivers
#
if publish.eligible:
# map eligible session IDs to eligible sessions
eligible = []
for session_id in publish.eligible:
if session_id in self._router._session_id_to_session:
eligible.append(
self._router.
_session_id_to_session[session_id])
# filter receivers for eligible sessions
receivers = set(eligible) & receivers
# remove "excluded" receivers
#
if publish.exclude:
# map excluded session IDs to excluded sessions
exclude = []
for s in publish.exclude:
if s in self._router._session_id_to_session:
exclude.append(
self._router._session_id_to_session[s])
# filter receivers for excluded sessions
if exclude:
receivers = receivers - set(exclude)
# if receivers is non-empty, dispatch event ..
#
receivers_cnt = len(receivers) - (1 if self
in receivers else 0)
if receivers_cnt:
# for pattern-based subscriptions, the EVENT must contain
# the actual topic being published to
#
if subscription.match != message.Subscribe.MATCH_EXACT:
topic = publish.topic
else:
topic = None
if publish.payload:
msg = message.Event(
subscription.id,
publication,
payload=publish.payload,
publisher=publisher,
publisher_authid=publisher_authid,
publisher_authrole=publisher_authrole,
topic=topic,
enc_algo=publish.enc_algo,
enc_key=publish.enc_key,
enc_serializer=publish.enc_serializer)
else:
msg = message.Event(
subscription.id,
publication,
args=publish.args,
kwargs=publish.kwargs,
publisher=publisher,
publisher_authid=publisher_authid,
publisher_authrole=publisher_authrole,
topic=topic)
for receiver in receivers:
if (me_also or receiver != session
) and receiver != self._event_store:
# the receiving subscriber session
# might have no transport, or no
# longer be joined
if receiver._session_id and receiver._transport:
self._router.send(receiver, msg)
def on_authorize_error(err):
"""
the call to authorize the action _itself_ failed (note this is
different from the call to authorize succeed, but the
authorization being denied)
"""
self.log.failure(err)
if publish.acknowledge:
reply = message.Error(
message.Publish.MESSAGE_TYPE, publish.request,
ApplicationError.AUTHORIZATION_FAILED, [
u"failed to authorize session for publishing to topic URI '{0}': {1}"
.format(publish.topic, err.value)
])
self._router.send(session, reply)
txaio.add_callbacks(d, on_authorize_success, on_authorize_error)
def processSubscribe(self, session, subscribe):
"""
Implements :func:`crossbar.router.interfaces.IBroker.processSubscribe`
"""
# check topic URI: for SUBSCRIBE, must be valid URI (either strict or loose), and all
# URI components must be non-empty for normal subscriptions, may be empty for
# wildcard subscriptions and must be non-empty for all but the last component for
# prefix subscriptions
#
if self._option_uri_strict:
if subscribe.match == u"wildcard":
uri_is_valid = _URI_PAT_STRICT_EMPTY.match(subscribe.topic)
elif subscribe.match == u"prefix":
uri_is_valid = _URI_PAT_STRICT_LAST_EMPTY.match(
subscribe.topic)
else:
uri_is_valid = _URI_PAT_STRICT_NON_EMPTY.match(subscribe.topic)
else:
if subscribe.match == u"wildcard":
uri_is_valid = _URI_PAT_LOOSE_EMPTY.match(subscribe.topic)
elif subscribe.match == u"prefix":
uri_is_valid = _URI_PAT_LOOSE_LAST_EMPTY.match(subscribe.topic)
else:
uri_is_valid = _URI_PAT_LOOSE_NON_EMPTY.match(subscribe.topic)
if not uri_is_valid:
reply = message.Error(
message.Subscribe.MESSAGE_TYPE, subscribe.request,
ApplicationError.INVALID_URI, [
u"subscribe for invalid topic URI '{0}'".format(
subscribe.topic)
])
self._router.send(session, reply)
return
# authorize SUBSCRIBE action
#
d = self._router.authorize(session, subscribe.topic, u'subscribe')
def on_authorize_success(authorization):
if not authorization[u'allow']:
# error reply since session is not authorized to subscribe
#
reply = message.Error(
message.Subscribe.MESSAGE_TYPE, subscribe.request,
ApplicationError.NOT_AUTHORIZED, [
u"session is not authorized to subscribe to topic '{0}'"
.format(subscribe.topic)
])
else:
# ok, session authorized to subscribe. now get the subscription
#
subscription, was_already_subscribed, is_first_subscriber = self._subscription_map.add_observer(
session, subscribe.topic, subscribe.match)
if not was_already_subscribed:
self._session_to_subscriptions[session].add(subscription)
# publish WAMP meta events
#
if self._router._realm:
service_session = self._router._realm.session
if service_session and not subscription.uri.startswith(
u'wamp.'):
if is_first_subscriber:
subscription_details = {
u'id': subscription.id,
u'created': subscription.created,
u'uri': subscription.uri,
u'match': subscription.match,
}
service_session.publish(
u'wamp.subscription.on_create',
session._session_id, subscription_details)
if not was_already_subscribed:
service_session.publish(
u'wamp.subscription.on_subscribe',
session._session_id, subscription.id)
# acknowledge subscribe with subscription ID
#
reply = message.Subscribed(subscribe.request, subscription.id)
# send out reply to subscribe requestor
#
self._router.send(session, reply)
def on_authorize_error(err):
"""
the call to authorize the action _itself_ failed (note this is
different from the call to authorize succeed, but the
authorization being denied)
"""
# XXX same as another code-block, can we collapse?
self.log.failure(err)
reply = message.Error(
message.Subscribe.MESSAGE_TYPE, subscribe.request,
ApplicationError.AUTHORIZATION_FAILED, [
u"failed to authorize session for subscribing to topic URI '{0}': {1}"
.format(subscribe.topic, err.value)
])
self._router.send(session, reply)
txaio.add_callbacks(d, on_authorize_success, on_authorize_error)
def processUnsubscribe(self, session, unsubscribe):
"""
Implements :func:`crossbar.router.interfaces.IBroker.processUnsubscribe`
"""
# get subscription by subscription ID or None (if it doesn't exist on this broker)
#
subscription = self._subscription_map.get_observation_by_id(
unsubscribe.subscription)
if subscription:
if session in subscription.observers:
was_subscribed, was_last_subscriber = self._unsubscribe(
subscription, session)
reply = message.Unsubscribed(unsubscribe.request)
else:
# subscription exists on this broker, but the session that wanted to unsubscribe wasn't subscribed
#
reply = message.Error(message.Unsubscribe.MESSAGE_TYPE,
unsubscribe.request,
ApplicationError.NO_SUCH_SUBSCRIPTION)
else:
# subscription doesn't even exist on this broker
#
reply = message.Error(message.Unsubscribe.MESSAGE_TYPE,
unsubscribe.request,
ApplicationError.NO_SUCH_SUBSCRIPTION)
self._router.send(session, reply)
def _unsubscribe(self, subscription, session):
# drop session from subscription observers
#
was_subscribed, was_last_subscriber = self._subscription_map.drop_observer(
session, subscription)
# remove subscription from session->subscriptions map
#
if was_subscribed:
self._session_to_subscriptions[session].discard(subscription)
# publish WAMP meta events
#
if self._router._realm:
service_session = self._router._realm.session
if service_session and not subscription.uri.startswith(u'wamp.'):
if was_subscribed:
service_session.publish(
u'wamp.subscription.on_unsubscribe',
session._session_id, subscription.id)
if was_last_subscriber:
service_session.publish(u'wamp.subscription.on_delete',
session._session_id,
subscription.id)
return was_subscribed, was_last_subscriber
def removeSubscriber(self, subscription, session, reason=None):
"""
Actively unsubscribe a subscriber session from a subscription.
"""
was_subscribed, was_last_subscriber = self._unsubscribe(
subscription, session)
if 'subscriber' in session._session_roles and session._session_roles[
'subscriber'] and session._session_roles[
'subscriber'].subscription_revocation:
reply = message.Unsubscribed(0,
subscription=subscription.id,
reason=reason)
self._router.send(session, reply)
return was_subscribed, was_last_subscriber
class TlsServerContextFactory(DefaultOpenSSLContextFactory):
"""
TLS context factory for use with Twisted.
Like the default
http://twistedmatrix.com/trac/browser/tags/releases/twisted-11.1.0/twisted/internet/ssl.py#L42
but loads key/cert from string, not file and supports chained certificates.
See also:
http://pyopenssl.sourceforge.net/pyOpenSSL.html/openssl-context.html
http://www.openssl.org/docs/ssl/SSL_CTX_use_certificate.html
Chained certificates:
The certificates must be in PEM format and must be sorted starting with
the subject's certificate (actual client or server certificate), followed
by intermediate CA certificates if applicable, and ending at the
highest level (root) CA.
Hardening:
http://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/
https://www.ssllabs.com/ssltest/analyze.html?d=www.example.com
"""
log = make_logger()
def __init__(self,
privateKeyString,
certificateString,
chainedCertificate=True,
dhParamFilename=None,
ciphers=None):
self._privateKeyString = str(privateKeyString).encode('utf8')
self._certificateString = str(certificateString).encode('utf8')
self._chainedCertificate = chainedCertificate
self._dhParamFilename = str(
dhParamFilename) if dhParamFilename else None
self._ciphers = str(ciphers) if ciphers else None
# do a SSLv2-compatible handshake even for TLS
#
self.sslmethod = SSL.SSLv23_METHOD
self._contextFactory = SSL.Context
self.cacheContext()
def cacheContext(self):
if self._context is None:
ctx = self._contextFactory(self.sslmethod)
# SSL hardening
#
ctx.set_options(SSL_DEFAULT_OPTIONS)
if self._ciphers:
ctx.set_cipher_list(self._ciphers)
self.log.info("Using explicit cipher list.")
else:
ctx.set_cipher_list(SSL_DEFAULT_CIPHERS)
self.log.info("Using default cipher list.")
# Activate DH(E)
#
# http://linux.die.net/man/3/ssl_ctx_set_tmp_dh
# http://linux.die.net/man/1/dhparam
#
if self._dhParamFilename:
try:
ctx.load_tmp_dh(self._dhParamFilename)
except Exception:
self.log.failure(
"Error: OpenSSL DH modes not active - failed to load DH parameter file [{log_failure}]"
)
else:
self.log.info(
"Ok, OpenSSL Diffie-Hellman ciphers parameter file loaded."
)
else:
self.log.warn(
"OpenSSL DH modes not active - missing DH param file")
# Activate ECDH(E)
#
# This needs pyOpenSSL 0.15
#
try:
# without setting a curve, ECDH won't be available even if listed
# in SSL_DEFAULT_CIPHERS!
# curve must be one of OpenSSL.crypto.get_elliptic_curves()
#
curve = crypto.get_elliptic_curve(ECDH_DEFAULT_CURVE_NAME)
ctx.set_tmp_ecdh(curve)
except Exception:
self.log.failure(
"Warning: OpenSSL failed to set ECDH default curve [{log_failure}]"
)
else:
self.log.info(
"Ok, OpenSSL is using ECDH elliptic curve {curve}",
curve=ECDH_DEFAULT_CURVE_NAME)
# load certificate (chain) into context
#
if not self._chainedCertificate:
cert = crypto.load_certificate(crypto.FILETYPE_PEM,
self._certificateString)
ctx.use_certificate(cert)
else:
# http://pyopenssl.sourceforge.net/pyOpenSSL.html/openssl-context.html
# there is no "use_certificate_chain" function, so we need to create
# a temporary file writing the certificate chain file
f = tempfile.NamedTemporaryFile(delete=False)
f.write(self._certificateString)
f.close()
ctx.use_certificate_chain_file(f.name)
# load private key into context
#
key = crypto.load_privatekey(crypto.FILETYPE_PEM,
self._privateKeyString)
ctx.use_privatekey(key)
ctx.check_privatekey()
# set cached context
#
self._context = ctx
def check_is_running(cbdir):
"""
Check if a Crossbar.io node is already running on a Crossbar.io node directory.
:param cbdir: The Crossbar.io node directory to check.
:type cbdir: str
:returns: The PID of the running Crossbar.io controller process or ``None``
:rtype: int or None
"""
log = make_logger()
remove_PID_type = None
remove_PID_reason = None
fp = os.path.join(cbdir, _PID_FILENAME)
if os.path.isfile(fp):
with open(fp) as fd:
pid_data_str = fd.read()
try:
pid_data = json.loads(pid_data_str)
pid = int(pid_data['pid'])
except ValueError:
remove_PID_type = "corrupt"
remove_PID_reason = "corrupt .pid file"
else:
if sys.platform == 'win32' and not _HAS_PSUTIL:
# when on Windows, and we can't actually determine if the PID exists,
# just assume it exists
return pid_data
else:
pid_exists = check_pid_exists(pid)
if pid_exists:
if _HAS_PSUTIL:
# additionally check this is actually a crossbar process
p = psutil.Process(pid)
cmdline = p.cmdline()
if not _is_crossbar_process(cmdline):
nicecmdline = ' '.join(cmdline)
if len(nicecmdline) > 76:
nicecmdline = nicecmdline[:
38] + ' ... ' + nicecmdline[
-38:]
log.info(
'"{}" points to PID {} which is not a crossbar process:'
.format(fp, pid))
log.info(' ' + nicecmdline)
log.info(
'Verify manually and either kill {} or delete {}'
.format(pid, fp))
return None
return pid_data
else:
remove_PID_type = "stale"
remove_PID_reason = "pointing to non-existing process with PID {}".format(
pid)
if remove_PID_type:
# If we have to remove a PID, do it here.
try:
os.remove(fp)
except:
log.info(("Could not remove {pidtype} Crossbar.io PID file "
"({reason}) {fp} - {log_failure}"),
pidtype=remove_PID_type,
reason=remove_PID_reason,
fp=fp)
else:
log.info("{pidtype} Crossbar.io PID file ({reason}) {fp} removed",
pidtype=remove_PID_type.title(),
reason=remove_PID_reason,
fp=fp)
return None
def run_command_version(options, reactor=None, **kwargs):
"""
Subcommand "crossbar version".
"""
log = make_logger()
# verbose = True
# Python
py_ver = '.'.join([str(x) for x in list(sys.version_info[:3])])
py_ver_string = "[%s]" % sys.version.replace('\n', ' ')
if 'pypy_version_info' in sys.__dict__:
py_ver_detail = "{}-{}".format(
platform.python_implementation(),
'.'.join(str(x) for x in sys.pypy_version_info[:3]))
else:
py_ver_detail = platform.python_implementation()
# Twisted / Reactor
tx_ver = "%s-%s" % (pkg_resources.require("Twisted")[0].version,
reactor.__class__.__name__)
tx_loc = "[%s]" % qual(reactor.__class__)
# txaio
txaio_ver = 'txaio-%s' % pkg_resources.require("txaio")[0].version
# Autobahn
from autobahn.websocket.protocol import WebSocketProtocol
ab_ver = pkg_resources.require("autobahn")[0].version
ab_loc = "[%s]" % qual(WebSocketProtocol)
# UTF8 Validator
from autobahn.websocket.utf8validator import Utf8Validator
s = qual(Utf8Validator)
if 'wsaccel' in s:
utf8_ver = 'wsaccel-%s' % pkg_resources.require('wsaccel')[0].version
elif s.startswith('autobahn'):
utf8_ver = 'autobahn'
else:
# could not detect UTF8 validator type/version
utf8_ver = '?'
utf8_loc = "[%s]" % qual(Utf8Validator)
# XOR Masker
from autobahn.websocket.xormasker import XorMaskerNull
s = qual(XorMaskerNull)
if 'wsaccel' in s:
xor_ver = 'wsaccel-%s' % pkg_resources.require('wsaccel')[0].version
elif s.startswith('autobahn'):
xor_ver = 'autobahn'
else:
# could not detect XOR masker type/version
xor_ver = '?'
xor_loc = "[%s]" % qual(XorMaskerNull)
# JSON Serializer
supported_serializers = ['JSON']
from autobahn.wamp.serializer import JsonObjectSerializer
s = str(JsonObjectSerializer.JSON_MODULE)
if 'ujson' in s:
json_ver = 'ujson-%s' % pkg_resources.require('ujson')[0].version
else:
json_ver = 'stdlib'
# MsgPack Serializer
try:
import msgpack # noqa
msgpack_ver = 'msgpack-python-%s' % pkg_resources.require(
'msgpack-python')[0].version
supported_serializers.append('MessagePack')
except ImportError:
msgpack_ver = '-'
# CBOR Serializer
try:
import cbor # noqa
cbor_ver = 'cbor-%s' % pkg_resources.require('cbor')[0].version
supported_serializers.append('CBOR')
except ImportError:
cbor_ver = '-'
# LMDB
try:
import lmdb # noqa
lmdb_lib_ver = '.'.join([str(x) for x in lmdb.version()])
lmdb_ver = '{}/lmdb-{}'.format(
pkg_resources.require('lmdb')[0].version, lmdb_lib_ver)
except ImportError:
lmdb_ver = '-'
def decorate(text):
return click.style(text, fg='yellow', bold=True)
for line in BANNER.splitlines():
log.info(decorate("{:>40}".format(line)))
pad = " " * 22
log.info(" Crossbar.io : {ver}", ver=decorate(crossbar.__version__))
log.info(" Autobahn : {ver} (with {serializers})",
ver=decorate(ab_ver),
serializers=', '.join(supported_serializers))
log.trace("{pad}{debuginfo}", pad=pad, debuginfo=decorate(ab_loc))
log.debug(" txaio : {ver}", ver=decorate(txaio_ver))
log.debug(" UTF8 Validator : {ver}", ver=decorate(utf8_ver))
log.trace("{pad}{debuginfo}", pad=pad, debuginfo=decorate(utf8_loc))
log.debug(" XOR Masker : {ver}", ver=decorate(xor_ver))
log.trace("{pad}{debuginfo}", pad=pad, debuginfo=decorate(xor_loc))
log.debug(" JSON Codec : {ver}", ver=decorate(json_ver))
log.debug(" MsgPack Codec : {ver}", ver=decorate(msgpack_ver))
log.debug(" CBOR Codec : {ver}", ver=decorate(cbor_ver))
log.info(" Twisted : {ver}", ver=decorate(tx_ver))
log.trace("{pad}{debuginfo}", pad=pad, debuginfo=decorate(tx_loc))
log.info(" LMDB : {ver}", ver=decorate(lmdb_ver))
log.info(" Python : {ver}/{impl}",
ver=decorate(py_ver),
impl=decorate(py_ver_detail))
log.trace("{pad}{debuginfo}", pad=pad, debuginfo=decorate(py_ver_string))
log.info(" OS : {ver}", ver=decorate(platform.platform()))
log.info(" Machine : {ver}", ver=decorate(platform.machine()))
log.info("")
class ListeningEndpointTests(TestCase):
log = make_logger()
def setUp(self):
self.cbdir = self.mktemp()
FilePath(self.cbdir).makedirs()
return super(ListeningEndpointTests, self).setUp()
def test_unix(self):
"""
A config with type = "unix" will create an endpoint for a UNIX socket
at the given path.
"""
path = FilePath("/tmp").child(uuid4().hex).path
self.addCleanup(os.remove, path)
reactor = SelectReactor()
config = {"type": "unix", "path": path}
endpoint = create_listening_endpoint_from_config(
config, self.cbdir, reactor, self.log)
self.assertTrue(isinstance(endpoint, UNIXServerEndpoint))
factory = Factory.forProtocol(Echo)
endpoint.listen(factory)
self.assertIn(
factory,
[getattr(x, "factory", None) for x in reactor.getReaders()])
def test_unix_already_listening(self):
"""
A config with type = "unix" will create an endpoint for a UNIX socket
at the given path, and delete it if required.
"""
path = FilePath("/tmp").child(uuid4().hex).path
self.addCleanup(os.remove, path)
# Something is already there
FilePath(path).setContent(b"")
reactor = SelectReactor()
config = {"type": "unix", "path": path}
endpoint = create_listening_endpoint_from_config(
config, self.cbdir, reactor, self.log)
self.assertTrue(isinstance(endpoint, UNIXServerEndpoint))
factory = Factory.forProtocol(Echo)
endpoint.listen(factory)
self.assertIn(
factory,
[getattr(x, "factory", None) for x in reactor.getReaders()])
def test_unix_already_listening_cant_delete(self):
"""
A config with type = "unix" will create an endpoint for a UNIX socket
at the given path, and delete it if required. If it can't delete it, it
will raise an exception.
"""
parent_fp = FilePath("/tmp").child(uuid4().hex)
parent_fp.makedirs()
fp = parent_fp.child(uuid4().hex)
# Something is already there
fp.setContent(b"")
fp.chmod(0o544)
parent_fp.chmod(0o544)
reactor = SelectReactor()
config = {"type": "unix", "path": fp.path}
with self.assertRaises(OSError) as e:
create_listening_endpoint_from_config(config, self.cbdir, reactor,
self.log)
self.assertEqual(e.exception.errno, 13) # Permission Denied
parent_fp.chmod(0o777)
parent_fp.remove()
if platform.isWindows():
_ = "Windows does not have UNIX sockets"
test_unix.skip = _
test_unix_already_listening.skip = _
test_unix_already_listening_cant_delete.skip = _
del _
from __future__ import absolute_import
import os
from twisted.python.failure import Failure
from autobahn.twisted import websocket
from autobahn.twisted import rawsocket
from autobahn.websocket.compress import * # noqa
import crossbar
from crossbar.router.cookiestore import CookieStoreMemoryBacked, CookieStoreFileBacked
from crossbar._logging import make_logger
log = make_logger()
__all__ = (
'WampWebSocketServerFactory',
'WampRawSocketServerFactory',
'WampWebSocketServerProtocol',
'WampRawSocketServerProtocol',
'WampWebSocketClientFactory',
'WampRawSocketClientFactory',
'WampWebSocketClientProtocol',
'WampRawSocketClientProtocol',
)
def set_websocket_options(factory, options):
"""
def run():
"""
Entry point into (native) worker processes. This wires up stuff such that
a worker instance is talking WAMP-over-stdio to the node controller.
"""
import os
import sys
import platform
import signal
# Ignore SIGINT so we get consistent behavior on control-C versus
# sending SIGINT to the controller process. When the controller is
# shutting down, it sends TERM to all its children but ctrl-C
# handling will send a SIGINT to all the processes in the group
# (so then the controller sends a TERM but the child already or
# will very shortly get a SIGINT as well). Twisted installs signal
# handlers, but not for SIGINT if there's already a custom one
# present.
def ignore(sig, frame):
log.debug("Ignoring SIGINT in worker.")
signal.signal(signal.SIGINT, ignore)
# create the top-level parser
#
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--reactor',
default=None,
choices=['select', 'poll', 'epoll', 'kqueue', 'iocp'],
help='Explicit Twisted reactor selection (optional).')
parser.add_argument('--loglevel',
default="info",
choices=['none', 'error', 'warn', 'info', 'debug', 'trace'],
help='Initial log level.')
parser.add_argument('-c',
'--cbdir',
type=str,
help="Crossbar.io node directory (required).")
parser.add_argument('-n',
'--node',
type=str,
help='Crossbar.io node ID (required).')
parser.add_argument('-w',
'--worker',
type=str,
help='Crossbar.io worker ID (required).')
parser.add_argument('-r',
'--realm',
type=str,
help='Crossbar.io node (management) realm (required).')
parser.add_argument('-t',
'--type',
choices=['router', 'container'],
help='Worker type (required).')
parser.add_argument('--title',
type=str,
default=None,
help='Worker process title to set (optional).')
options = parser.parse_args()
# make sure logging to something else than stdio is setup _first_
#
from crossbar._logging import make_JSON_observer, cb_logging_aware, _stderr
from crossbar._logging import make_logger, log_publisher, start_logging
from crossbar._logging import set_global_log_level
# Set the global log level
set_global_log_level(options.loglevel)
log = make_logger()
# Print a magic phrase that tells the capturing logger that it supports
# Crossbar's rich logging
print(cb_logging_aware, file=_stderr)
_stderr.flush()
flo = make_JSON_observer(_stderr)
log_publisher.addObserver(flo)
start_logging()
try:
import setproctitle
except ImportError:
log.debug("Could not set worker process title (setproctitle not installed)")
else:
# set process title if requested to
#
if options.title:
setproctitle.setproctitle(options.title)
else:
WORKER_TYPE_TO_TITLE = {
'router': 'crossbar-worker [router]',
'container': 'crossbar-worker [container]'
}
setproctitle.setproctitle(WORKER_TYPE_TO_TITLE[options.type].strip())
# we use an Autobahn utility to import the "best" available Twisted reactor
#
from autobahn.twisted.choosereactor import install_reactor
reactor = install_reactor(options.reactor)
from twisted.python.reflect import qual
log.info("Worker running under {python}-{reactor}",
python=platform.python_implementation(),
reactor=qual(reactor.__class__).split('.')[-1])
options.cbdir = os.path.abspath(options.cbdir)
os.chdir(options.cbdir)
# log.msg("Starting from node directory {}".format(options.cbdir))
from crossbar.worker.router import RouterWorkerSession
from crossbar.worker.container import ContainerWorkerSession
WORKER_TYPE_TO_CLASS = {
'router': RouterWorkerSession,
'container': ContainerWorkerSession
}
from autobahn.twisted.websocket import WampWebSocketServerProtocol
class WorkerServerProtocol(WampWebSocketServerProtocol):
def connectionLost(self, reason):
try:
# this log message is unlikely to reach the controller (unless
# only stdin/stdout pipes were lost, but not stderr)
log.warn("Connection to node controller lost.")
WampWebSocketServerProtocol.connectionLost(self, reason)
except:
pass
finally:
# losing the connection to the node controller is fatal:
# stop the reactor and exit with error
log.info("No more controller connection; shutting down.")
reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1)
try:
reactor.stop()
except ReactorNotRunning:
pass
try:
# create a WAMP application session factory
#
from autobahn.twisted.wamp import ApplicationSessionFactory
from autobahn.wamp.types import ComponentConfig
session_config = ComponentConfig(realm=options.realm, extra=options)
session_factory = ApplicationSessionFactory(session_config)
session_factory.session = WORKER_TYPE_TO_CLASS[options.type]
# create a WAMP-over-WebSocket transport server factory
#
from autobahn.twisted.websocket import WampWebSocketServerFactory
transport_factory = WampWebSocketServerFactory(session_factory, "ws://localhost", debug=False, debug_wamp=False)
transport_factory.protocol = WorkerServerProtocol
transport_factory.setProtocolOptions(failByDrop=False)
# create a protocol instance and wire up to stdio
#
from twisted.python.runtime import platform as _platform
from twisted.internet import stdio
proto = transport_factory.buildProtocol(None)
if _platform.isWindows():
stdio.StandardIO(proto)
else:
stdio.StandardIO(proto, stdout=3)
# now start reactor loop
#
if False:
log.info("vmprof enabled.")
import os
import vmprof
PROFILE_FILE = 'vmprof_{}.dat'.format(os.getpid())
outfd = os.open(PROFILE_FILE, os.O_RDWR | os.O_CREAT | os.O_TRUNC)
vmprof.enable(outfd, period=0.01)
log.info("Entering event loop...")
reactor.run()
vmprof.disable()
else:
log.debug("Entering event loop...")
reactor.run()
except Exception as e:
log.info("Unhandled exception: {}".format(e))
if reactor.running:
reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1)
reactor.stop()
else:
sys.exit(1)
class Router(object):
"""
Crossbar.io core router class.
"""
log = make_logger()
RESERVED_ROLES = [u'trusted']
"""
Roles with these URIs are built-in and cannot be added/dropped.
"""
broker = Broker
"""
The broker class this router will use.
"""
dealer = Dealer
"""
The dealer class this router will use.
"""
def __init__(self, factory, realm, options=None, store=None):
"""
:param factory: The router factory this router was created by.
:type factory: Object that implements :class:`autobahn.wamp.interfaces.IRouterFactory`..
:param realm: The realm this router is working for.
:type realm: str
:param options: Router options.
:type options: Instance of :class:`autobahn.wamp.types.RouterOptions`.
"""
self._factory = factory
self._options = options or RouterOptions()
self._store = store
self._realm = realm
self.realm = realm.config[u'name']
self._trace_traffic = False
self._trace_traffic_roles_include = None
self._trace_traffic_roles_exclude = [u'trusted']
# map: session_id -> session
self._session_id_to_session = {}
self._broker = self.broker(self, self._options)
self._dealer = self.dealer(self, self._options)
self._attached = 0
self._roles = {u'trusted': RouterTrustedRole(self, u'trusted')}
def attach(self, session):
"""
Implements :func:`autobahn.wamp.interfaces.IRouter.attach`
"""
if session._session_id not in self._session_id_to_session:
if _is_client_session(session):
self._session_id_to_session[session._session_id] = session
else:
self.log.debug("attaching non-client session {session}",
session=session)
else:
raise Exception("session with ID {} already attached".format(
session._session_id))
self._broker.attach(session)
self._dealer.attach(session)
self._attached += 1
return {
u'broker': self._broker._role_features,
u'dealer': self._dealer._role_features
}
def detach(self, session):
"""
Implements :func:`autobahn.wamp.interfaces.IRouter.detach`
"""
self._broker.detach(session)
self._dealer.detach(session)
if session._session_id in self._session_id_to_session:
del self._session_id_to_session[session._session_id]
else:
if _is_client_session(session):
raise Exception("session with ID {} not attached".format(
session._session_id))
self._attached -= 1
if not self._attached:
self._factory.onLastDetach(self)
def _check_trace(self, session, msg):
if not self._trace_traffic:
return False
if self._trace_traffic_roles_include and session._authrole not in self._trace_traffic_roles_include:
return False
if self._trace_traffic_roles_exclude and session._authrole in self._trace_traffic_roles_exclude:
return False
return True
def send(self, session, msg):
if self._check_trace(session, msg):
self.log.info("<<TX<< {msg}", msg=msg)
session._transport.send(msg)
def process(self, session, msg):
"""
Implements :func:`autobahn.wamp.interfaces.IRouter.process`
"""
if self._check_trace(session, msg):
self.log.info(">>RX>> {msg}", msg=msg)
# Broker
#
if isinstance(msg, message.Publish):
self._broker.processPublish(session, msg)
elif isinstance(msg, message.Subscribe):
self._broker.processSubscribe(session, msg)
elif isinstance(msg, message.Unsubscribe):
self._broker.processUnsubscribe(session, msg)
# Dealer
#
elif isinstance(msg, message.Register):
self._dealer.processRegister(session, msg)
elif isinstance(msg, message.Unregister):
self._dealer.processUnregister(session, msg)
elif isinstance(msg, message.Call):
self._dealer.processCall(session, msg)
elif isinstance(msg, message.Cancel):
self._dealer.processCancel(session, msg)
elif isinstance(msg, message.Yield):
self._dealer.processYield(session, msg)
elif isinstance(
msg, message.Error
) and msg.request_type == message.Invocation.MESSAGE_TYPE:
self._dealer.processInvocationError(session, msg)
else:
raise ProtocolError("Unexpected message {0}".format(msg.__class__))
def has_role(self, uri):
"""
Check if a role with given URI exists on this router.
:returns: bool - `True` if a role under the given URI exists on this router.
"""
return uri in self._roles
def add_role(self, role):
"""
Adds a role to this router.
:param role: The role to add.
:type role: An instance of :class:`crossbar.router.session.CrossbarRouterRole`.
:returns: bool -- `True` if a role under the given URI actually existed before and was overwritten.
"""
self.log.debug("CrossbarRouter.add_role({role})", role=role)
if role.uri in self.RESERVED_ROLES:
raise Exception("cannot add reserved role '{}'".format(role.uri))
overwritten = role.uri in self._roles
self._roles[role.uri] = role
return overwritten
def drop_role(self, role):
"""
Drops a role from this router.
:param role: The role to drop.
:type role: An instance of :class:`crossbar.router.session.CrossbarRouterRole`.
:returns: bool -- `True` if a role under the given URI actually existed and was removed.
"""
self.log.debug("CrossbarRouter.drop_role({role})", role=role)
if role.uri in self.RESERVED_ROLES:
raise Exception("cannot drop reserved role '{}'".format(role.uri))
if role.uri in self._roles:
del self._roles[role.uri]
return True
else:
return False
def authorize(self, session, uri, action):
"""
Authorizes a session for an action on an URI.
Implements :func:`autobahn.wamp.interfaces.IRouter.authorize`
"""
assert (type(uri) == six.text_type)
assert (action in [u'call', u'register', u'publish', u'subscribe'])
# the role under which the session that wishes to perform the given action on
# the given URI was authenticated under
role = session._authrole
if role in self._roles:
# the authorizer procedure of the role which we will call ..
authorize = self._roles[role].authorize
d = txaio.as_future(authorize, session, uri, action)
else:
# normally, the role should exist on the router (and hence we should not arrive
# here), but the role might have been dynamically removed - and anyway, safety first!
d = txaio.create_future_success(False)
def got_authorization(authorization):
# backward compatibility
if type(authorization) == bool:
authorization = {u'allow': authorization, u'cache': False}
if action in [u'call', u'publish']:
authorization[u'disclose'] = False
self.log.info(
"Authorized action '{action}' for URI '{uri}' by session {session_id} with authid '{authid}' and authrole '{authrole}' -> authorization: {authorization}",
session_id=session._session_id,
uri=uri,
action=action,
authid=session._authid,
authrole=session._authrole,
authorization=authorization)
return authorization
d.addCallback(got_authorization)
return d
def validate(self, payload_type, uri, args, kwargs):
"""
Implements :func:`autobahn.wamp.interfaces.IRouter.validate`
"""
self.log.debug("Validate '{payload_type}' for '{uri}'",
payload_type=payload_type,
uri=uri,
cb_level="trace")
class NodeManagementBridgeSession(ApplicationSession):
"""
The management bridge is a WAMP session that lives on the local management router,
but has access to a 2nd WAMP session that lives on the uplink CDC router.
The bridge is responsible for forwarding calls from CDC into the local node,
and for forwarding events from the local node to CDC.
"""
log = make_logger()
def __init__(self, config, management_session):
"""
:param config: Session configuration.
:type config: instance of `autobahn.wamp.types.ComponentConfig`
:param management_session: uplink session.
:type management_session: instance of `autobahn.wamp.protocol.ApplicationSession`
"""
ApplicationSession.__init__(self, config)
self._management_session = management_session
self._regs = {}
@inlineCallbacks
def onJoin(self, details):
self.log.debug("Joined realm '{realm}' on node management router", realm=details.realm)
@inlineCallbacks
def on_event(*args, **kwargs):
details = kwargs.pop('details')
topic = u"cdc." + details.topic
try:
yield self._management_session.publish(topic, *args, options=PublishOptions(acknowledge=True), **kwargs)
except Exception as e:
self.log.error(e)
else:
self.log.debug("Forwarded event on topic '{topic}'", topic=topic)
yield self.subscribe(on_event, u"crossbar.node", options=SubscribeOptions(match=u"prefix", details_arg="details"))
# we use the WAMP meta API implemented by CB to get notified whenever a procedure is
# registered/unregister on the node management router, setup a forwarding procedure
# and register that on the uplink CDC router
#
@inlineCallbacks
def on_registration_create(session_id, registration):
uri = registration['uri']
def forward_call(*args, **kwargs):
return self.call(uri, *args, **kwargs)
reg = yield self._management_session.register(forward_call, uri)
self._regs[registration['id']] = reg
self.log.debug("Forwarding procedure: {procedure}", procedure=reg.procedure)
yield self.subscribe(on_registration_create, u'wamp.registration.on_create')
@inlineCallbacks
def on_registration_delete(session_id, registration_id):
reg = self._regs.pop(registration_id, None)
if reg:
yield reg.unregister()
self.log.debug("Removed forwarding of procedure {procedure}", procedure=reg.procedure)
else:
self.log.warn("Could not remove forwarding for unmapped registration_id {reg_id}", reg_id=registration_id)
yield self.subscribe(on_registration_delete, u'wamp.registration.on_delete')
self.log.info("Management bridge ready")
class FileUploadResource(Resource):
"""
Twisted Web resource that handles file uploads over `HTTP/POST` requests.
"""
log = make_logger()
def __init__(self,
upload_directory,
temp_directory,
form_fields,
upload_session,
options=None):
"""
:param upload_directory: The target directory where uploaded files will be stored.
:type upload_directory: str
:param temp_directory: A temporary directory where chunks of a file being uploaded are stored.
:type temp_directory: str
:param form_fields: Names of HTML form fields used for uploading.
:type form_fields: dict
:param upload_session: An instance of `ApplicationSession` used for publishing progress events.
:type upload_session: obj
:param options: Options for file upload.
:type options: dict or None
"""
Resource.__init__(self)
self._dir = upload_directory
self._tempDir = temp_directory
self._form_fields = form_fields
self._fileupload_session = upload_session
self._options = options or {}
self._max_file_size = self._options.get('max_file_size',
10 * 1024 * 1024)
self._fileTypes = self._options.get('file_types', None)
self._file_permissions = self._options.get('file_permissions', None)
# track uploaded files / chunks
self._uploads = {}
# scan the temp dir for uploaded chunks and fill the _uploads dict with it
# so existing uploads can be resumed
for fileTempDir in os.listdir(self._tempDir):
ft = os.path.join(self._tempDir, fileTempDir)
if os.path.isdir(ft):
self._uploads[fileTempDir] = {
'chunk_list': {},
'origin': 'startup'
}
for chunk in os.listdir(ft):
if chunk[:6] == 'chunk_':
self._uploads[fileTempDir]['chunk_list'][int(
chunk[6:])] = True
self.log.debug("Scanned pending uploads: {uploads}",
uploads=self._uploads)
def render_POST(self, request):
headers = request.getAllHeaders()
origin = headers['host']
content = cgi.FieldStorage(fp=request.content,
headers=headers,
environ={
'REQUEST_METHOD': 'POST',
'CONTENT_TYPE': headers['content-type']
})
f = self._form_fields
filename = content[f['file_name']].value
totalSize = int(content[f['total_size']].value)
totalChunks = int(content[f['total_chunks']].value)
chunkSize = int(content[f['chunk_size']].value)
chunkNumber = int(content[f['chunk_number']].value)
fileContent = content[f['content']].value
fileId = filename
# # prepare user specific upload areas
# # NOT YET IMPLEMENTED
# #
# if 'auth_id' in f and f['auth_id'] in content:
# auth_id = content[f['auth_id']].value
# mydir = os.path.join(self._dir, auth_id)
# my_temp_dir = os.path.join(self._tempDir, auth_id)
#
# # check if auth_id is a valid directory_name
# #
# if auth_id != auth_id.encode('ascii', 'ignore'):
# msg = "The requestor auth_id must be an ascii string."
# if self._debug:
# log.msg(msg)
# # 415 Unsupported Media Type
# request.setResponseCode(415, msg)
# return msg
# else:
# auth_id = 'anonymous'
# create user specific folder
# mydir = self._dir
# my_temp_dir = self._tempDir
# if not os.path.exists(mydir):
# os.makedirs(mydir)
# if not os.path.exists(my_temp_dir):
# os.makedirs(my_temp_dir)
if 'on_progress' in f and f[
'on_progress'] in content and self._fileupload_session != {}:
topic = content[f['on_progress']].value
if 'session' in f and f['session'] in content:
session = int(content[f['session']].value)
publish_options = PublishOptions(eligible=[session])
else:
publish_options = None
def fileupload_publish(payload):
self._fileupload_session.publish(topic,
payload,
options=publish_options)
else:
def fileupload_publish(payload):
pass
# Register upload right at the start to avoid overlapping upload conflicts
if fileId not in self._uploads:
self._uploads[fileId] = {'chunk_list': {}, 'origin': origin}
chunk_is_first = True
else:
chunk_is_first = False
self.log.debug(
'Started upload of file: file_name={file_name}, total_size={total_size}, total_chunks={total_chunks}, chunk_size={chunk_size}, chunk_number={chunk_number}',
file_name=fileId,
total_size=totalSize,
total_chunks=totalChunks,
chunk_size=chunkSize,
chunk_number=chunkNumber)
# check file size
#
if totalSize > self._max_file_size:
msg = "Size {} of file to be uploaded exceeds maximum {}".format(
totalSize, self._max_file_size)
self.log.debug(msg)
# 413 Request Entity Too Large
request.setResponseCode(413, msg)
return msg
# check file extensions
#
extension = os.path.splitext(filename)[1]
if self._fileTypes and extension not in self._fileTypes:
msg = "Type '{}' of file to be uploaded is in allowed types {}".format(
extension, self._fileTypes)
self.log.debug(msg)
# 415 Unsupported Media Type
request.setResponseCode(415, msg)
return msg
# check if another session is uploading this file already
# If the chunks are read at startup of crossbar any client may resume the pending upload !
#
try:
upl = self._uploads[fileId]
if upl['origin'] != origin and upl['origin'] != 'startup':
msg = "File being uploaded is already uploaded in a different session"
self.log.debug(msg)
# 409 Conflict
request.setResponseCode(409, msg)
return msg
except Exception:
pass
# TODO: check mime type
fileTempDir = os.path.join(self._tempDir, fileId)
chunkName = os.path.join(fileTempDir, 'chunk_' + str(chunkNumber))
_chunkName = os.path.join(
fileTempDir,
'#kfhfkzuru578e38viokbjhfvz4w__' + 'chunk_' + str(chunkNumber))
if chunk_is_first:
# first chunk of file
# clean the temp dir once per file upload
self._remove_stale_uploads()
# publish file upload start
#
fileupload_publish({
"id": fileId,
"chunk": chunkNumber,
"name": filename,
"total": totalSize,
"remaining": totalSize,
"status": "started",
"progress": 0.
})
if totalChunks == 1:
# only one chunk overall -> write file directly
finalFileName = os.path.join(self._dir, fileId)
_finalFileName = os.path.join(
self._dir, '#kfhfkzuru578e38viokbjhfvz4w__' + fileId)
with open(_finalFileName, 'wb') as finalFile:
finalFile.write(fileContent)
os.rename(_finalFileName, finalFileName)
self._uploads[fileId]['chunk_list'][chunkNumber] = True
if self._file_permissions:
perm = int(self._file_permissions, 8)
try:
os.chmod(finalFileName, perm)
except Exception as e:
os.remove(finalFileName)
msg = "Could not change file permissions of uploaded file"
self.log.debug(msg)
self.log.debug(e)
request.setResponseCode(500, msg)
return msg
else:
self.log.debug(
"Changed permissions on {file_name} to {permissions}",
file_name=finalFileName,
permissions=self._file_permissions)
self._uploads.pop(fileId, None)
# publish file upload progress to file_progress_URI
fileupload_publish({
"id": fileId,
"chunk": chunkNumber,
"name": filename,
"total": totalSize,
"remaining": 0,
"status": "finished",
"progress": 1.
})
else:
# first of more chunks
os.makedirs(fileTempDir)
with open(_chunkName, 'wb') as chunk:
chunk.write(fileContent)
os.rename(_chunkName, chunkName)
self._uploads[fileId]['chunk_list'][chunkNumber] = True
# publish file upload progress
#
fileupload_publish({
"id":
fileId,
"chunk":
chunkNumber,
"name":
filename,
"total":
totalSize,
"remaining":
totalSize - chunkSize,
"status":
"progress",
"progress":
round(float(chunkSize) / float(totalSize), 3)
})
else:
# intermediate chunk
with open(_chunkName, 'wb') as chunk:
chunk.write(fileContent)
os.rename(_chunkName, chunkName)
self._uploads[fileId]['chunk_list'][chunkNumber] = True
received = sum(
os.path.getsize(os.path.join(fileTempDir, f))
for f in os.listdir(fileTempDir))
fileupload_publish({
"id":
fileId,
"chunk":
chunkNumber,
"name":
filename,
"total":
totalSize,
"remaining":
totalSize - received,
"status":
"progress",
"progress":
round(float(received) / float(totalSize), 3)
})
# every chunk has to check if it is the last chunk written, except in a single chunk scenario
if totalChunks > 1 and len(
self._uploads[fileId]['chunk_list']) == totalChunks:
# last chunk
self.log.debug('Finished file upload after chunk {chunk_number}',
chunk_number=chunkNumber)
# Merge all files into one file and remove the temp files
# TODO: How to avoid the extra file IO ?
finalFileName = os.path.join(self._dir, fileId)
_finalFileName = os.path.join(
self._dir, '#kfhf3kz412uru578e38viokbjhfvz4w__' + fileId)
with open(_finalFileName, 'wb') as finalFile:
for tfileName in os.listdir(fileTempDir):
with open(os.path.join(fileTempDir, tfileName),
'r') as tfile:
finalFile.write(tfile.read())
os.rename(_finalFileName, finalFileName)
if self._file_permissions:
perm = int(self._file_permissions, 8)
try:
os.chmod(finalFileName, perm)
except Exception as e:
msg = "file upload resource - could not change file permissions of uploaded file"
self.log.debug(msg)
self.log.debug(e)
request.setResponseCode(500, msg)
return msg
else:
self.log.debug(
"Changed permissions on {file_name} to {permissions}",
file_name=finalFileName,
permissions=self._file_permissions)
# publish file upload progress to file_progress_URI
fileupload_publish({
"id": fileId,
"chunk": chunkNumber,
"name": filename,
"total": totalSize,
"remaining": 0,
"status": "finished",
"progress": 1.
})
# remove the file temp folder
self._remove_temp_dir(fileTempDir)
self._uploads.pop(fileId, None)
request.setResponseCode(200)
return ''
def _remove_temp_dir(self, fileTempDir):
for tfileName in os.listdir(fileTempDir):
os.remove(os.path.join(fileTempDir, tfileName))
os.rmdir(fileTempDir)
def _remove_stale_uploads(self):
"""
This only works if there is a temp folder exclusive for crossbar file uploads
if the system temp folder is used then crossbar creates a "crossbar-uploads" there and
uses that as the temp folder for uploads
If you don't clean up regularly an attacker could fill up the OS file system
"""
for _dir in os.listdir(self._tempDir):
fileTempDir = os.path.join(self._tempDir, _dir)
if os.path.isdir(fileTempDir) and _dir not in self._uploads:
self._remove_temp_dir(fileTempDir)
def render_GET(self, request):
"""
This method can be used to check wether a chunk has been uploaded already.
It returns with HTTP status code `200` if yes and `404` if not.
The request needs to contain the file identifier and the chunk number to check for.
"""
for param in ['file_name', 'chunk_number']:
if not self._form_fields[param] in request.args:
msg = "file upload resource - missing request query parameter '{}', configured from '{}'".format(
self._form_fields[param], param)
self.log.debug(msg)
# 400 Bad Request
request.setResponseCode(400, msg)
return msg
file_name = request.args[self._form_fields['file_name']][0]
chunk_number = int(request.args[self._form_fields['chunk_number']][0])
# a complete upload will be repeated an incomplete upload will be resumed
if file_name in self._uploads and chunk_number in self._uploads[
file_name]['chunk_list']:
self.log.debug(
"Skipping chunk upload {file_name} of chunk {chunk_number}",
file_name=file_name,
chunk_number=chunk_number)
msg = "chunk of file already uploaded"
request.setResponseCode(200, msg)
return msg
else:
msg = "chunk of file not yet uploaded"
request.setResponseCode(404, msg)
return msg
class WampLongPollResourceSessionReceive(Resource):
"""
A Web resource for receiving via XHR that is part of :class:`autobahn.twisted.longpoll.WampLongPollResourceSession`.
"""
log = make_logger()
def __init__(self, parent):
"""
:param parent: The Web parent resource for the WAMP session.
:type parent: Instance of :class:`autobahn.twisted.longpoll.WampLongPollResourceSession`.
"""
Resource.__init__(self)
self._parent = parent
self.reactor = self._parent._parent.reactor
self._queue = deque()
self._request = None
self._killed = False
# FIXME: can we read the loglevel from self.log currently set?
if False:
def logqueue():
if not self._killed:
self.log.debug(
"WampLongPoll: transport '{0}' - currently polled {1}, pending messages {2}"
.format(self._parent._transport_id, self._request
is not None, len(self._queue)))
self.reactor.callLater(1, logqueue)
logqueue()
def queue(self, data):
"""
Enqueue data to be received by client.
:param data: The data to be received by the client.
:type data: bytes
"""
self._queue.append(data)
self._trigger()
def _kill(self):
"""
Kill any outstanding request.
"""
if self._request:
self._request.finish()
self._request = None
self._killed = True
def _trigger(self):
"""
Trigger batched sending of queued messages.
"""
if self._request and len(self._queue):
if self._parent._serializer._serializer._batched:
# in batched mode, write all pending messages
while len(self._queue) > 0:
msg = self._queue.popleft()
self._request.write(msg)
else:
# in unbatched mode, only write 1 pending message
msg = self._queue.popleft()
if type(msg) == six.binary_type:
self._request.write(msg)
else:
self.log.error(
"internal error: cannot write data of type {type_} - {msg}",
type_=type(msg),
msg=msg,
)
self._request.finish()
self._request = None
def render_POST(self, request):
"""
A client receives WAMP messages by issuing a HTTP/POST to this
Web resource. The request will immediately return when there are
messages pending to be received. When there are no such messages
pending, the request will "just hang", until either a message
arrives to be received or a timeout occurs.
"""
# remember request, which marks the session as being polled
self._request = request
self._parent._parent._setStandardHeaders(request)
mime_type = self._parent._serializer.MIME_TYPE
if type(mime_type) == six.text_type:
mime_type = mime_type.encode('utf8')
request.setHeader(b'content-type', mime_type)
def cancel(_):
self.log.debug(
"WampLongPoll: poll request for transport '{0}' has gone away".
format(self._parent._transport_id))
self._request = None
request.notifyFinish().addErrback(cancel)
self._parent._isalive = True
self._trigger()
return NOT_DONE_YET
class WampLongPollResource(Resource):
"""
A WAMP-over-Longpoll resource for use with Twisted Web Resource trees.
This class provides an implementation of the
`WAMP-over-Longpoll Transport <https://github.com/tavendo/WAMP/blob/master/spec/advanced.md#long-poll-transport>`_
for WAMP.
The Resource exposes the following paths (child resources).
Opening a new WAMP session:
* ``<base-url>/open``
Once a transport is created and the session is opened:
* ``<base-url>/<transport-id>/send``
* ``<base-url>/<transport-id>/receive``
* ``<base-url>/<transport-id>/close``
"""
log = make_logger()
protocol = WampLongPollResourceSession
def __init__(self,
factory,
serializers=None,
timeout=10,
killAfter=30,
queueLimitBytes=128 * 1024,
queueLimitMessages=100,
debug_transport_id=None,
reactor=None):
"""
Create new HTTP WAMP Web resource.
:param factory: A (router) session factory.
:type factory: Instance of :class:`autobahn.twisted.wamp.RouterSessionFactory`.
:param serializers: List of WAMP serializers.
:type serializers: list of obj (which implement :class:`autobahn.wamp.interfaces.ISerializer`)
:param timeout: XHR polling timeout in seconds.
:type timeout: int
:param killAfter: Kill WAMP session after inactivity in seconds.
:type killAfter: int
:param queueLimitBytes: Kill WAMP session after accumulation of this many bytes in send queue (XHR poll).
:type queueLimitBytes: int
:param queueLimitMessages: Kill WAMP session after accumulation of this many message in send queue (XHR poll).
:type queueLimitMessages: int
:param debug: Enable debug logging.
:type debug: bool
:param debug_transport_id: If given, use this fixed transport ID.
:type debug_transport_id: str
:param reactor: The Twisted reactor to run under.
:type reactor: obj
"""
Resource.__init__(self)
# RouterSessionFactory
self._factory = factory
# lazy import to avoid reactor install upon module import
if reactor is None:
from twisted.internet import reactor
self.reactor = reactor
self._debug_transport_id = debug_transport_id
self._timeout = timeout
self._killAfter = killAfter
self._queueLimitBytes = queueLimitBytes
self._queueLimitMessages = queueLimitMessages
if serializers is None:
serializers = []
# try MsgPack WAMP serializer
try:
from autobahn.wamp.serializer import MsgPackSerializer
serializers.append(MsgPackSerializer(batched=True))
serializers.append(MsgPackSerializer())
except ImportError:
pass
# try JSON WAMP serializer
try:
from autobahn.wamp.serializer import JsonSerializer
serializers.append(JsonSerializer(batched=True))
serializers.append(JsonSerializer())
except ImportError:
pass
if not serializers:
raise Exception("could not import any WAMP serializers")
self._serializers = {}
for ser in serializers:
self._serializers[ser.SERIALIZER_ID] = ser
self._transports = {}
# <Base URL>/open
#
self.putChild(b"open", WampLongPollResourceOpen(self))
self.log.debug("WampLongPollResource initialized")
def render_GET(self, request):
request.setHeader(b'content-type', b'text/html; charset=UTF-8')
peer = b"{0}:{1}".format(request.client.host, request.client.port)
return self.getNotice(peer=peer)
def getChild(self, name, request):
"""
Returns send/receive/close resource for transport.
.. seealso::
* :class:`twisted.web.resource.Resource`
* :class:`zipfile.ZipFile`
"""
if name not in self._transports:
if name == b'':
return self
else:
return NoResource("no WAMP transport '{0}'".format(name))
if len(request.postpath) != 1 or request.postpath[0] not in [
b'send', b'receive', b'close'
]:
return NoResource("invalid WAMP transport operation '{0}'".format(
request.postpath))
return self._transports[name]
def _setStandardHeaders(self, request):
"""
Set standard HTTP response headers.
"""
origin = request.getHeader(b"origin")
if origin is None or origin == b"null":
origin = b"*"
request.setHeader(b'access-control-allow-origin', origin)
request.setHeader(b'access-control-allow-credentials', b'true')
request.setHeader(b'cache-control',
b'no-store, no-cache, must-revalidate, max-age=0')
headers = request.getHeader(b'access-control-request-headers')
if headers is not None:
request.setHeader(b'access-control-allow-headers', headers)
def _failRequest(self, request, msg):
"""
Fails a request to the long-poll service.
"""
self._setStandardHeaders(request)
request.setHeader(b'content-type', b'text/plain; charset=UTF-8')
request.setResponseCode(http.BAD_REQUEST)
return msg
def getNotice(self, peer, redirectUrl=None, redirectAfter=0):
"""
Render a user notice (HTML page) when the Long-Poll root resource
is accessed via HTTP/GET (by a user).
:param redirectUrl: Optional URL to redirect the user to.
:type redirectUrl: str
:param redirectAfter: When ``redirectUrl`` is provided, redirect after this time (seconds).
:type redirectAfter: int
"""
from autobahn import __version__
if redirectUrl:
redirect = b"""<meta http-equiv="refresh" content="%d;URL='%s'">""" % (
redirectAfter, redirectUrl)
else:
redirect = b""
html = b"""
<!DOCTYPE html>
<html>
<head>
%s
<style>
body {
color: #fff;
background-color: #027eae;
font-family: "Segoe UI", "Lucida Grande", "Helvetica Neue", Helvetica, Arial, sans-serif;
font-size: 16px;
}
a, a:visited, a:hover {
color: #fff;
}
</style>
</head>
<body>
<h1>AutobahnPython %s</h1>
<p>
I am not Web server, but a <b>WAMP-over-LongPoll</b> transport endpoint.
</p>
<p>
You can talk to me using the <a href="https://github.com/tavendo/WAMP/blob/master/spec/advanced.md#long-poll-transport">WAMP-over-LongPoll</a> protocol.
</p>
<p>
For more information, please see:
<ul>
<li><a href="http://wamp.ws/">WAMP</a></li>
<li><a href="http://autobahn.ws/python">AutobahnPython</a></li>
</ul>
</p>
</body>
</html>
""" % (redirect, __version__)
return html
class WampLongPollResourceSession(Resource):
"""
A Web resource representing an open WAMP session.
"""
log = make_logger()
def __init__(self, parent, transport_details):
"""
Create a new Web resource representing a WAMP session.
:param parent: The parent Web resource.
:type parent: Instance of :class:`autobahn.twisted.longpoll.WampLongPollResource`.
:param transport_details: Details on the WAMP-over-Longpoll transport session.
:type transport_details: dict
"""
Resource.__init__(self)
self._parent = parent
self.reactor = self._parent.reactor
self._transport_id = transport_details['transport']
self._serializer = transport_details['serializer']
self._session = None
# session authentication information
#
self._authid = None
self._authrole = None
self._authmethod = None
self._authprovider = None
self._send = WampLongPollResourceSessionSend(self)
self._receive = WampLongPollResourceSessionReceive(self)
self._close = WampLongPollResourceSessionClose(self)
self.putChild(b"send", self._send)
self.putChild(b"receive", self._receive)
self.putChild(b"close", self._close)
self._isalive = False
# kill inactive sessions after this timeout
#
killAfter = self._parent._killAfter
if killAfter > 0:
def killIfDead():
if not self._isalive:
self.log.debug(
"WampLongPoll: killing inactive WAMP session with transport '{0}'"
.format(self._transport_id))
self.onClose(False, 5000, "session inactive")
self._receive._kill()
if self._transport_id in self._parent._transports:
del self._parent._transports[self._transport_id]
else:
self.log.debug(
"WampLongPoll: transport '{0}' is still alive".format(
self._transport_id))
self._isalive = False
self.reactor.callLater(killAfter, killIfDead)
self.reactor.callLater(killAfter, killIfDead)
else:
self.log.debug(
"WampLongPoll: transport '{0}' automatic killing of inactive session disabled"
.format(self._transport_id))
self.log.debug(
"WampLongPoll: session resource for transport '{0}' initialized)".
format(self._transport_id))
self.onOpen()
def close(self):
"""
Implements :func:`autobahn.wamp.interfaces.ITransport.close`
"""
if self.isOpen():
self.onClose(True, 1000, u"session closed")
self._receive._kill()
del self._parent._transports[self._transport_id]
else:
raise TransportLost()
def abort(self):
"""
Implements :func:`autobahn.wamp.interfaces.ITransport.abort`
"""
if self.isOpen():
self.onClose(True, 1000, u"session aborted")
self._receive._kill()
del self._parent._transports[self._transport_id]
else:
raise TransportLost()
# noinspection PyUnusedLocal
def onClose(self, wasClean, code, reason):
"""
Callback from :func:`autobahn.websocket.interfaces.IWebSocketChannel.onClose`
"""
if self._session:
try:
self._session.onClose(wasClean)
except Exception:
# ignore exceptions raised here, but log ..
self.log.failure("invoking session's onClose failed")
self._session = None
def onOpen(self):
"""
Callback from :func:`autobahn.websocket.interfaces.IWebSocketChannel.onOpen`
"""
self._session = self._parent._factory()
# noinspection PyBroadException
try:
self._session.onOpen(self)
except Exception:
# ignore exceptions raised here, but log ..
self.log.failure("Invoking session's onOpen failed")
def onMessage(self, payload, isBinary):
"""
Callback from :func:`autobahn.websocket.interfaces.IWebSocketChannel.onMessage`
"""
for msg in self._serializer.unserialize(payload, isBinary):
self.log.debug("WampLongPoll: RX {0}".format(msg))
self._session.onMessage(msg)
def send(self, msg):
"""
Implements :func:`autobahn.wamp.interfaces.ITransport.send`
"""
if self.isOpen():
try:
self.log.debug("WampLongPoll: TX {0}".format(msg))
payload, isBinary = self._serializer.serialize(msg)
except Exception as e:
# all exceptions raised from above should be serialization errors ..
raise SerializationError(
"unable to serialize WAMP application payload ({0})".
format(e))
else:
self._receive.queue(payload)
else:
raise TransportLost()
def isOpen(self):
"""
Implements :func:`autobahn.wamp.interfaces.ITransport.isOpen`
"""
return self._session is not None
class RouterServiceSession(ApplicationSession):
"""
Router service session which is used internally by a router to
issue WAMP calls or publish events, and which provides WAMP meta API
procedures.
"""
log = make_logger()
def __init__(self, config, router, schemas=None):
"""
Ctor.
:param config: WAMP application component configuration.
:type config: Instance of :class:`autobahn.wamp.types.ComponentConfig`.
:param router: The router this service session is running for.
:type: router: instance of :class:`crossbar.router.session.CrossbarRouter`
:param schemas: An (optional) initial schema dictionary to load.
:type schemas: dict
"""
ApplicationSession.__init__(self, config)
self._router = router
self._schemas = {}
if schemas:
self._schemas.update(schemas)
self.log.info("initialized schemas cache with {} entries".format(
len(self._schemas)))
@inlineCallbacks
def onJoin(self, details):
self.log.debug("Router service session attached: {}".format(details))
regs = yield self.register(self)
self.log.debug("Registered {} procedures".format(len(regs)))
@wamp.register(u'wamp.session.list')
def session_list(self, filter_authroles=None):
"""
Get list of session IDs of sessions currently joined on the router.
:param filter_authroles: If provided, only return sessions with an authrole from this list.
:type filter_authroles: None or list
:returns: List of WAMP session IDs (order undefined).
:rtype: list
"""
assert (filter_authroles is None or type(filter_authroles) == list)
session_ids = []
for session in self._router._session_id_to_session.values():
if not is_restricted_session(session):
if filter_authroles is None or session._session_details[
'authrole'] in filter_authroles:
session_ids.append(session._session_id)
return session_ids
@wamp.register(u'wamp.session.count')
def session_count(self, filter_authroles=None):
"""
Count sessions currently joined on the router.
:param filter_authroles: If provided, only count sessions with an authrole from this list.
:type filter_authroles: None or list
:returns: Count of joined sessions.
:rtype: int
"""
assert (filter_authroles is None or type(filter_authroles) == list)
session_count = 0
for session in self._router._session_id_to_session.values():
if not is_restricted_session(session):
if filter_authroles is None or session._session_details[
'authrole'] in filter_authroles:
session_count += 1
return session_count
@wamp.register(u'wamp.session.get')
def session_get(self, session_id):
"""
Get details for given session.
:param session_id: The WAMP session ID to retrieve details for.
:type session_id: int
:returns: WAMP session details.
:rtype: dict or None
"""
if session_id in self._router._session_id_to_session:
session = self._router._session_id_to_session[session_id]
if not is_restricted_session(session):
return session._session_details
raise ApplicationError(
ApplicationError.NO_SUCH_SESSION,
message="no session with ID {} exists on this router".format(
session_id))
@wamp.register(u'wamp.session.kill')
def session_kill(self, session_id, reason=None, message=None):
"""
Forcefully kill a session.
:param session_id: The WAMP session ID of the session to kill.
:type session_id: int
:param reason: A reason URI provided to the killed session.
:type reason: unicode or None
"""
if session_id in self._router._session_id_to_session:
session = self._router._session_id_to_session[session_id]
if not is_restricted_session(session):
session.leave(reason=reason, message=message)
return
raise ApplicationError(
ApplicationError.NO_SUCH_SESSION,
message="no session with ID {} exists on this router".format(
session_id))
@wamp.register(u'wamp.registration.remove_callee')
def registration_remove_callee(self,
registration_id,
callee_id,
reason=None):
"""
Forcefully remove callee from registration.
:param registration_id: The ID of the registration to remove the callee from.
:type registration_id: int
:param callee_id: The WAMP session ID of the callee to remove.
:type callee_id: int
"""
callee = self._router._session_id_to_session.get(callee_id, None)
if not callee:
raise ApplicationError(
ApplicationError.NO_SUCH_SESSION,
message="no session with ID {} exists on this router".format(
callee_id))
registration = self._router._dealer._registration_map.get_observation_by_id(
registration_id)
if registration and not is_protected_uri(registration.uri):
if callee not in registration.observers:
raise ApplicationError(
ApplicationError.NO_SUCH_REGISTRATION,
message=
"session {} is not registered on registration {} on this dealer"
.format(callee_id, registration_id))
self._router._dealer.removeCallee(registration,
callee,
reason=reason)
else:
raise ApplicationError(
ApplicationError.NO_SUCH_REGISTRATION,
message="no registration with ID {} exists on this dealer".
format(registration_id))
@wamp.register(u'wamp.subscription.remove_subscriber')
def subscription_remove_subscriber(self,
subscription_id,
subscriber_id,
reason=None):
"""
Forcefully remove subscriber from subscription.
:param subscription_id: The ID of the subscription to remove the subscriber from.
:type subscription_id: int
:param subscriber_id: The WAMP session ID of the subscriber to remove.
:type subscriber_id: int
"""
subscriber = self._router._session_id_to_session.get(
subscriber_id, None)
if not subscriber:
raise ApplicationError(
ApplicationError.NO_SUCH_SESSION,
message="no session with ID {} exists on this router".format(
subscriber_id))
subscription = self._router._broker._subscription_map.get_observation_by_id(
subscription_id)
if subscription and not is_protected_uri(subscription.uri):
if subscriber not in subscription.observers:
raise ApplicationError(
ApplicationError.NO_SUCH_SUBSCRIPTION,
message=
"session {} is not subscribed on subscription {} on this broker"
.format(subscriber_id, subscription_id))
self._router._broker.removeSubscriber(subscription,
subscriber,
reason=reason)
else:
raise ApplicationError(
ApplicationError.NO_SUCH_SUBSCRIPTION,
message="no subscription with ID {} exists on this broker".
format(subscription_id))
@wamp.register(u'wamp.registration.get')
def registration_get(self, registration_id):
"""
Get registration details.
:param registration_id: The ID of the registration to retrieve.
:type registration_id: int
:returns: The registration details.
:rtype: dict
"""
registration = self._router._dealer._registration_map.get_observation_by_id(
registration_id)
if registration and not is_protected_uri(registration.uri):
registration_details = {
'id': registration.id,
'created': registration.created,
'uri': registration.uri,
'match': registration.match,
'invoke': registration.extra.invoke,
}
return registration_details
else:
raise ApplicationError(
ApplicationError.NO_SUCH_REGISTRATION,
message="no registration with ID {} exists on this dealer".
format(registration_id))
@wamp.register(u'wamp.subscription.get')
def subscription_get(self, subscription_id):
"""
Get subscription details.
:param subscription_id: The ID of the subscription to retrieve.
:type subscription_id: int
:returns: The subscription details.
:rtype: dict
"""
subscription = self._router._broker._subscription_map.get_observation_by_id(
subscription_id)
if subscription and not is_protected_uri(subscription.uri):
subscription_details = {
'id': subscription.id,
'created': subscription.created,
'uri': subscription.uri,
'match': subscription.match,
}
return subscription_details
else:
raise ApplicationError(
ApplicationError.NO_SUCH_SUBSCRIPTION,
message="no subscription with ID {} exists on this broker".
format(subscription_id))
@wamp.register(u'wamp.registration.list')
def registration_list(self):
"""
List current registrations.
:returns: A dictionary with three entries for the match policies 'exact', 'prefix'
and 'wildcard', with a list of registration IDs for each.
:rtype: dict
"""
registration_map = self._router._dealer._registration_map
registrations_exact = []
for registration in registration_map._observations_exact.values():
if not is_protected_uri(registration.uri):
registrations_exact.append(registration.id)
registrations_prefix = []
for registration in registration_map._observations_prefix.values():
if not is_protected_uri(registration.uri):
registrations_prefix.append(registration.id)
registrations_wildcard = []
for registration in registration_map._observations_wildcard.values():
if not is_protected_uri(registration.uri):
registrations_wildcard.append(registration.id)
return {
'exact': registrations_exact,
'prefix': registrations_prefix,
'wildcard': registrations_wildcard,
}
@wamp.register(u'wamp.subscription.list')
def subscription_list(self):
"""
List current subscriptions.
:returns: A dictionary with three entries for the match policies 'exact', 'prefix'
and 'wildcard', with a list of subscription IDs for each.
:rtype: dict
"""
subscription_map = self._router._broker._subscription_map
subscriptions_exact = []
for subscription in subscription_map._observations_exact.values():
if not is_protected_uri(subscription.uri):
subscriptions_exact.append(subscription.id)
subscriptions_prefix = []
for subscription in subscription_map._observations_prefix.values():
if not is_protected_uri(subscription.uri):
subscriptions_prefix.append(subscription.id)
subscriptions_wildcard = []
for subscription in subscription_map._observations_wildcard.values():
if not is_protected_uri(subscription.uri):
subscriptions_wildcard.append(subscription.id)
return {
'exact': subscriptions_exact,
'prefix': subscriptions_prefix,
'wildcard': subscriptions_wildcard,
}
@wamp.register(u'wamp.registration.match')
def registration_match(self, procedure):
"""
Given a procedure URI, return the registration best matching the procedure.
This essentially models what a dealer does for dispatching an incoming call.
:param procedure: The procedure to match.
:type procedure: unicode
:returns: The best matching registration or ``None``.
:rtype: obj or None
"""
registration = self._router._dealer._registration_map.best_matching_observation(
procedure)
if registration and not is_protected_uri(registration.uri):
return registration.id
else:
return None
@wamp.register(u'wamp.subscription.match')
def subscription_match(self, topic):
"""
Given a topic URI, returns all subscriptions matching the topic.
This essentially models what a broker does for dispatching an incoming publication.
:param topic: The topic to match.
:type topic: unicode
:returns: All matching subscriptions or ``None``.
:rtype: obj or None
"""
subscriptions = self._router._broker._subscription_map.match_observations(
topic)
if subscriptions:
subscription_ids = []
for subscription in subscriptions:
if not is_protected_uri(subscription.uri):
subscription_ids.append(subscription.id)
if subscription_ids:
return subscription_ids
else:
return None
else:
return None
@wamp.register(u'wamp.registration.lookup')
def registration_lookup(self, procedure, options=None):
"""
Given a procedure URI (and options), return the registration (if any) managing the procedure.
This essentially models what a dealer does when registering for a procedure.
:param procedure: The procedure to lookup the registration for.
:type procedure: unicode
:param options: Same options as when registering a procedure.
:type options: dict or None
:returns: The ID of the registration managing the procedure or ``None``.
:rtype: int or None
"""
options = options or {}
match = options.get('match', u'exact')
registration = self._router._dealer._registration_map.get_observation(
procedure, match)
if registration and not is_protected_uri(registration.uri):
return registration.id
else:
return None
@wamp.register(u'wamp.subscription.lookup')
def subscription_lookup(self, topic, options=None):
"""
Given a topic URI (and options), return the subscription (if any) managing the topic.
This essentially models what a broker does when subscribing for a topic.
:param topic: The topic to lookup the subscription for.
:type topic: unicode
:param options: Same options as when subscribing to a topic.
:type options: dict or None
:returns: The ID of the subscription managing the topic or ``None``.
:rtype: int or None
"""
options = options or {}
match = options.get('match', u'exact')
subscription = self._router._broker._subscription_map.get_observation(
topic, match)
if subscription and not is_protected_uri(subscription.uri):
return subscription.id
else:
return None
@wamp.register(u'wamp.registration.list_callees')
def registration_list_callees(self, registration_id):
"""
Retrieve list of callees (WAMP session IDs) registered on (attached to) a registration.
:param registration_id: The ID of the registration to get callees for.
:type registration_id: int
:returns: A list of WAMP session IDs of callees currently attached to the registration.
:rtype: list
"""
registration = self._router._dealer._registration_map.get_observation_by_id(
registration_id)
if registration and not is_protected_uri(registration.uri):
session_ids = []
for callee in registration.observers:
session_ids.append(callee._session_id)
return session_ids
else:
raise ApplicationError(
ApplicationError.NO_SUCH_REGISTRATION,
message="no registration with ID {} exists on this dealer".
format(registration_id))
@wamp.register(u'wamp.subscription.list_subscribers')
def subscription_list_subscribers(self, subscription_id):
"""
Retrieve list of subscribers (WAMP session IDs) subscribed on (attached to) a subscription.
:param subscription_id: The ID of the subscription to get subscribers for.
:type subscription_id: int
:returns: A list of WAMP session IDs of subscribers currently attached to the subscription.
:rtype: list
"""
subscription = self._router._broker._subscription_map.get_observation_by_id(
subscription_id)
if subscription and not is_protected_uri(subscription.uri):
session_ids = []
for subscriber in subscription.observers:
session_ids.append(subscriber._session_id)
return session_ids
else:
raise ApplicationError(
ApplicationError.NO_SUCH_SUBSCRIPTION,
message="no subscription with ID {} exists on this broker".
format(subscription_id))
@wamp.register(u'wamp.registration.count_callees')
def registration_count_callees(self, registration_id):
"""
Retrieve number of callees registered on (attached to) a registration.
:param registration_id: The ID of the registration to get the number of callees for.
:type registration_id: int
:returns: Number of callees currently attached to the registration.
:rtype: int
"""
registration = self._router._dealer._registration_map.get_observation_by_id(
registration_id)
if registration and not is_protected_uri(registration.uri):
return len(registration.observers)
else:
raise ApplicationError(
ApplicationError.NO_SUCH_REGISTRATION,
message="no registration with ID {} exists on this dealer".
format(registration_id))
@wamp.register(u'wamp.subscription.count_subscribers')
def subscription_count_subscribers(self, subscription_id):
"""
Retrieve number of subscribers subscribed on (attached to) a subscription.
:param subscription_id: The ID of the subscription to get the number subscribers for.
:type subscription_id: int
:returns: Number of subscribers currently attached to the subscription.
:rtype: int
"""
subscription = self._router._broker._subscription_map.get_observation_by_id(
subscription_id)
if subscription and not is_protected_uri(subscription.uri):
return len(subscription.observers)
else:
raise ApplicationError(
ApplicationError.NO_SUCH_SUBSCRIPTION,
message="no subscription with ID {} exists on this broker".
format(subscription_id))
@wamp.register(u'wamp.subscription.get_events')
def subscription_get_events(self, subscription_id, limit=10):
"""
Return history of events for given subscription.
:param subscription_id: The ID of the subscription to get events for.
:type subscription_id: int
:param limit: Return at most this many events.
:type limit: int
:returns: List of events.
:rtype: list
"""
self.log.info("subscription_get_events({subscription_id}, {limit})",
subscription_id=subscription_id,
limit=limit)
subscription = self._router._broker._subscription_map.get_observation_by_id(
subscription_id)
if subscription and not is_protected_uri(subscription.uri):
events = self._router._broker._event_store.get_events(
subscription_id, limit)
if events is None:
raise Exception(
"event history for the given subscription is unavailable")
raise ApplicationError(
u'wamp.error.history_unavailable',
message=
"event history for the given subscription is unavailable")
else:
return events
else:
raise ApplicationError(
ApplicationError.NO_SUCH_SUBSCRIPTION,
message="no subscription with ID {} exists on this broker".
format(subscription_id))
@wamp.register(u'wamp.test.exception')
def test_exception(self):
raise ApplicationError(u'wamp.error.history_unavailable')
@wamp.register(u'wamp.schema.describe')
def schema_describe(self, uri=None):
"""
Describe a given URI or all URIs.
:param uri: The URI to describe or ``None`` to retrieve all declarations.
:type uri: unicode
:returns: A list of WAMP schema declarations.
:rtype: list
"""
if uri:
return self._schemas.get(uri, None)
else:
return self._schemas
@wamp.register(u'wamp.schema.define')
def schema_define(self, uri, schema):
"""
Declare metadata for a given URI.
:param uri: The URI for which to declare metadata.
:type uri: unicode
:param schema: The WAMP schema declaration for
the URI or `None` to remove any declarations for the URI.
:type schema: dict
:returns: ``None`` if declaration was unchanged, ``True`` if
declaration was new, ``False`` if declaration existed, but was modified.
:rtype: bool or None
"""
if not schema:
if uri in self._schemas:
del self._schemas
self.publish(u'wamp.schema.on_undefine', uri)
return uri
else:
return None
if uri not in self._schemas:
was_new = True
was_modified = False
else:
was_new = False
if json.dumps(schema) != json.dumps(self._schemas[uri]):
was_modified = True
else:
was_modified = False
if was_new or was_modified:
self._schemas[uri] = schema
self.publish(u'wamp.schema.on_define', uri, schema, was_new)
return was_new
else:
return None
def __init__(self):
self.log = make_logger("info", logger=Mock)
class MemoryEventStore(object):
"""
Event store in-memory implementation.
"""
log = make_logger()
def __init__(self, config=None):
# whole store configuration
self._config = config or {}
# limit to event history per subscription
self._limit = self._config.get('limit', 1000)
# map of publication ID -> event dict
self._event_store = {}
# map of publication ID -> set of subscription IDs
self._event_subscriptions = {}
# map of subscription ID -> deque of publication IDs
self._event_history = {}
def attach_subscription_map(self, subscription_map):
# example topic being configured as persistent
for sub in self._config.get('event-history', []):
# FIXME: limit = sub.get('limit', self._limit)
subscription_map.add_observer(self,
uri=sub['uri'],
match=sub.get('match', u'exact'))
def store_event(self,
publisher_id,
publication_id,
topic,
args=None,
kwargs=None):
"""
Persist the given event to history.
:param publisher_id: The session ID of the publisher of the event being persisted.
:type publisher_id: int
:param publication_id: The publication ID of the event.
:type publisher_id: int
:param topic: The topic URI of the event.
:type topic: unicode
:param args: The args payload of the event.
:type args: list or None
:param kwargs: The kwargs payload of the event.
:type kwargs: dict or None
"""
assert (publication_id not in self._event_store)
evt = {
'timestamp': utcnow(),
'publisher': publisher_id,
'publication': publication_id,
'topic': topic,
'args': args,
'kwargs': kwargs
}
self._event_store[publication_id] = evt
self.log.debug("event {publication_id} persisted",
publication_id=publication_id)
def store_event_history(self, publication_id, subscription_id):
"""
Persist the given publication history to subscriptions.
:param publication_id: The ID of the event publication to be persisted.
:type publication_id: int
:param subscription_id: The ID of the subscription the event (identified by the publication ID),
was published to, because the event's topic matched the subscription.
:type subscription_id: int
"""
assert (publication_id in self._event_store)
# for in-memory history, we just use a double-ended queue
if subscription_id not in self._event_history:
self._event_history[subscription_id] = deque()
# append event to history
self._event_history[subscription_id].append(publication_id)
if publication_id not in self._event_subscriptions:
self._event_subscriptions[publication_id] = set()
self._event_subscriptions[publication_id].add(subscription_id)
self.log.debug(
"event {publication_id} history persisted for subscription {subscription_id}",
publication_id=publication_id,
subscription_id=subscription_id)
# purge history if over limit
if len(self._event_history[subscription_id]) > self._limit:
# remove leftmost event from history
purged_publication_id = self._event_history[
subscription_id].popleft()
# remove the purged publication from event subscriptions
self._event_subscriptions[purged_publication_id].remove(
subscription_id)
self.log.debug(
"event {publication_id} purged fom history for subscription {subscription_id}",
publication_id=purged_publication_id,
subscription_id=subscription_id)
# if no more event subscriptions exist for publication, remove that too
if not self._event_subscriptions[purged_publication_id]:
del self._event_subscriptions[purged_publication_id]
del self._event_store[purged_publication_id]
self.log.debug("event {publication_id} purged completey",
publication_id=purged_publication_id)
def get_events(self, subscription_id, limit):
"""
Retrieve given number of last events for a given subscription.
If no history is maintained for the given subscription, None is returned.
:param subscription_id: The ID of the subscription to retrieve events for.
:type subscription_id: int
:param limit: Limit number of events returned.
:type limit: int
:return: List of events.
:rtype: list or None
"""
if subscription_id not in self._event_history:
return None
else:
s = self._event_history[subscription_id]
# at most "limit" events in reverse chronological order
res = []
i = -1
if limit > len(s):
limit = len(s)
for _ in range(limit):
res.append(self._event_store[s[i]])
i -= 1
return res
def get_event_history(self, subscription_id, from_ts, until_ts):
"""
Retrieve event history for time range for a given subscription.
If no history is maintained for the given subscription, None is returned.
:param subscription_id: The ID of the subscription to retrieve events for.
:type subscription_id: int
:param from_ts: Filter events from this date (string in ISO-8601 format).
:type from_ts: unicode
:param until_ts: Filter events until this date (string in ISO-8601 format).
:type until_ts: unicode
"""
raise Exception("not implemented")
class RouterFactory(object):
"""
Crossbar.io core router factory.
"""
log = make_logger()
router = Router
"""
The router class this factory will create router instances from.
"""
def __init__(self, node_id, options=None):
"""
:param options: Default router options.
:type options: Instance of :class:`autobahn.wamp.types.RouterOptions`.
"""
assert (type(node_id) == six.text_type)
self._node_id = node_id
self._routers = {}
self._options = options or RouterOptions(
uri_check=RouterOptions.URI_CHECK_LOOSE)
self._auto_create_realms = False
def get(self, realm):
"""
Implements :func:`autobahn.wamp.interfaces.IRouterFactory.get`
"""
if self._auto_create_realms:
if realm not in self._routers:
self._routers[realm] = self.router(self, realm, self._options)
self.log.debug("Router created for realm '{realm}'",
realm=realm)
return self._routers[realm]
else:
return self._routers[realm]
def __getitem__(self, realm):
return self._routers[realm]
def __contains__(self, realm):
return realm in self._routers
def onLastDetach(self, router):
assert (router.realm in self._routers)
del self._routers[router.realm]
self.log.debug("Router destroyed for realm '{realm}'",
realm=router.realm)
def start_realm(self, realm):
"""
Starts a realm on this router.
:param realm: The realm to start.
:type realm: instance of :class:`crossbar.worker.router.RouterRealm`.
:returns: The router instance for the started realm.
:rtype: instance of :class:`crossbar.router.session.CrossbarRouter`
"""
self.log.debug("CrossbarRouterFactory.start_realm(realm = {realm})",
realm=realm)
# get name of realm (an URI in general)
#
uri = realm.config['name']
assert (uri not in self._routers)
# if configuration of realm contains a "store" item, set up a
# realm store as appropriate ..
store = None
if 'store' in realm.config:
store_config = realm.config['store']
if store_config['type'] == 'lmdb':
# if LMDB is available, and a realm store / database is configured,
# create an LMDB environment
if not HAS_LMDB:
raise Exception("LDMB not available")
store = LmdbRealmStore(store_config)
elif store_config['type'] == 'memory':
store = MemoryRealmStore(store_config)
else:
raise Exception('logic error')
# now create a router for the realm
#
router = Router(self, realm, self._options, store=store)
self._routers[uri] = router
self.log.debug("Router created for realm '{uri}'", uri=uri)
return router
def stop_realm(self, realm):
self.log.debug("CrossbarRouterFactory.stop_realm(realm = {realm})",
realm=realm)
def add_role(self, realm, config):
self.log.debug(
"CrossbarRouterFactory.add_role(realm = {realm}, config = {config})",
realm=realm,
config=config)
assert (type(realm) == six.text_type)
assert (realm in self._routers)
router = self._routers[realm]
uri = config[u'name']
if u'permissions' in config:
role = RouterRoleStaticAuth(router, uri, config[u'permissions'])
elif u'authorizer' in config:
role = RouterRoleDynamicAuth(router, uri, config[u'authorizer'])
else:
allow_by_default = config.get(u'allow-by-default', False)
role = RouterRole(router, uri, allow_by_default=allow_by_default)
router.add_role(role)
def drop_role(self, realm, role):
self.log.debug(
"CrossbarRouterFactory.drop_role(realm = {realm}, role = {role})",
realm=realm,
role=role)
class JsonResource(Resource):
"""
Static Twisted Web resource that renders to a JSON document.
"""
log = make_logger()
def __init__(self, value, options=None):
Resource.__init__(self)
options = options or {}
if options.get('prettify', False):
self._data = json.dumps(value,
sort_keys=True,
indent=3,
ensure_ascii=False)
else:
self._data = json.dumps(value,
separators=(',', ':'),
ensure_ascii=False)
# Twisted Web render_METHOD methods are expected to return a byte string
self._data = self._data.encode('utf8')
self._allow_cross_origin = options.get('allow_cross_origin', True)
self._discourage_caching = options.get('discourage_caching', False)
# number of HTTP/GET requests we served from this resource
#
self._requests_served = 0
def render_GET(self, request):
# we produce JSON: set correct response content type
#
# note: both args to request.setHeader are supposed to be byte strings
# https://twistedmatrix.com/documents/current/api/twisted.web.http.Request.html#setHeader
#
request.setHeader(b'content-type', b'application/json; charset=utf8-8')
# set response headers for cross-origin requests
#
if self._allow_cross_origin:
origin = request.getHeader(b'origin')
if origin is None or origin == b'null':
origin = b'*'
request.setHeader(b'access-control-allow-origin', origin)
request.setHeader(b'access-control-allow-credentials', b'true')
headers = request.getHeader(b'access-control-request-headers')
if headers is not None:
request.setHeader(b'access-control-allow-headers', headers)
# set response headers to disallow caching
#
if self._discourage_caching:
request.setHeader(
b'cache-control',
b'no-store, no-cache, must-revalidate, max-age=0')
self._requests_served += 1
if self._requests_served % 10000 == 0:
self.log.debug("Served {requests_served} requests",
requests_served=self._requests_served)
return self._data
class NodeControllerSession(NativeProcessSession):
"""
Singleton node WAMP session hooked up to the node management router.
This class exposes the node's management API.
"""
log = make_logger()
def __init__(self, node):
"""
:param node: The node singleton for this node controller session.
:type node: obj
"""
NativeProcessSession.__init__(self, reactor=node._reactor)
# associated node
self._node = node
self._node_id = node._node_id
self._realm = node._realm
self.cbdir = self._node._cbdir
self._started = None
self._pid = os.getpid()
# map of worker processes: worker_id -> NativeWorkerProcess
self._workers = {}
self._shutdown_requested = False
def onConnect(self):
self.log.debug("Connected to node management router")
# self._uri_prefix = u'crossbar.node.{}'.format(self.config.extra.node)
self._uri_prefix = u'crossbar.node.{}'.format(self._node_id)
NativeProcessSession.onConnect(self, False)
# self.join(self.config.realm)
self.join(self._realm)
@inlineCallbacks
def onJoin(self, details):
self.log.info("Joined realm '{realm}' on node management router",
realm=details.realm)
# When a (native) worker process has connected back to the router of
# the node controller, the worker will publish this event
# to signal it's readyness.
#
def on_worker_ready(res):
id = res['id']
if id in self._workers:
ready = self._workers[id].ready
if not ready.called:
# fire the Deferred previously stored for
# signaling "worker ready"
ready.callback(id)
else:
self.log.error(
"Internal error: on_worker_ready() fired for process {process}, but already called earlier",
process=id)
else:
self.log.error(
"Internal error: on_worker_ready() fired for process {process}, but no process with that ID",
process=id)
self.subscribe(
on_worker_ready,
'crossbar.node.{}.on_worker_ready'.format(self._node_id))
yield NativeProcessSession.onJoin(self, details)
# register node controller procedures: 'crossbar.node.<ID>.<PROCEDURE>'
#
procs = [
'get_info',
'shutdown',
'get_workers',
'get_worker_log',
'start_router',
'stop_router',
'start_container',
'stop_container',
'start_guest',
'stop_guest',
'start_websocket_testee',
'stop_websocket_testee',
]
dl = []
for proc in procs:
uri = '{}.{}'.format(self._uri_prefix, proc)
self.log.debug("Registering management API procedure {proc}",
proc=uri)
dl.append(
self.register(getattr(self, proc),
uri,
options=RegisterOptions(details_arg='details')))
regs = yield DeferredList(dl)
self.log.debug("Registered {cnt} management API procedures",
cnt=len(regs))
self._started = utcnow()
self.publish(u"crossbar.node.on_ready", self._node_id)
self.log.debug("Node controller ready")
def get_info(self, details=None):
"""
Return basic information about this node.
:returns: Information on the Crossbar.io node.
:rtype: dict
"""
return {
'started': self._started,
'pid': self._pid,
'workers': len(self._workers),
'directory': self.cbdir,
'wamplets': self._get_wamplets()
}
@inlineCallbacks
def shutdown(self, restart=False, mode=None, details=None):
"""
Stop this node.
"""
if self._shutdown_requested:
# we're already shutting down .. ignore ..
return
self._shutdown_requested = True
self.log.warn("Shutting down node...")
# publish management API event
shutdown_info = {
'restart': restart,
'mode': mode,
'who': details.caller if details else None,
'when': utcnow()
}
yield self.publish('crossbar.node.{}.on_shutdown'.format(
self._node_id),
shutdown_info,
options=PublishOptions(
exclude=[details.caller] if details else None,
acknowledge=True))
def stop_reactor():
try:
self._reactor.stop()
except ReactorNotRunning:
pass
self._reactor.callLater(0, stop_reactor)
returnValue(shutdown_info)
def _get_wamplets(self):
"""
List installed WAMPlets.
"""
res = []
for entrypoint in pkg_resources.iter_entry_points(
'autobahn.twisted.wamplet'):
try:
e = entrypoint.load()
except Exception as e:
pass
else:
ep = {}
ep['dist'] = entrypoint.dist.key
ep['version'] = entrypoint.dist.version
ep['location'] = entrypoint.dist.location
ep['name'] = entrypoint.name
ep['module_name'] = entrypoint.module_name
ep['entry_point'] = str(entrypoint)
if hasattr(e, '__doc__') and e.__doc__:
ep['doc'] = e.__doc__.strip()
else:
ep['doc'] = None
ep['meta'] = e(None)
res.append(ep)
return sorted(res)
def get_workers(self, details=None):
"""
Returns the list of workers currently running on this node.
:returns: List of worker processes.
:rtype: list of dicts
"""
now = datetime.utcnow()
res = []
for worker in sorted(self._workers.values(), key=lambda w: w.created):
res.append({
'id':
worker.id,
'pid':
worker.pid,
'type':
worker.TYPE,
'status':
worker.status,
'created':
utcstr(worker.created),
'started':
utcstr(worker.started),
'startup_time':
(worker.started -
worker.created).total_seconds() if worker.started else None,
'uptime':
(now -
worker.started).total_seconds() if worker.started else None,
})
return res
def get_worker_log(self, id, limit=None, details=None):
"""
Get buffered log for a worker.
:param limit: Optionally, limit the amount of log entries returned
to the last N entries.
:type limit: None or int
:return: Buffered log for worker.
:rtype: list
"""
if id not in self._workers:
emsg = "No worker with ID '{}'".format(id)
raise ApplicationError(u'crossbar.error.no_such_worker', emsg)
return self._workers[id].getlog(limit)
def start_router(self, id, options=None, details=None):
"""
Start a new router worker: a Crossbar.io native worker process
that runs a WAMP router.
:param id: The worker ID to start this router with.
:type id: str
:param options: The router worker options.
:type options: dict
"""
self.log.debug(
"NodeControllerSession.start_router({id}, options={options})",
id=id,
options=options)
return self._start_native_worker('router',
id,
options,
details=details)
def start_container(self, id, options=None, details=None):
"""
Start a new container worker: a Crossbar.io native worker process
that can host WAMP application components written in Python.
:param id: The worker ID to start this container with.
:type id: str
:param options: The container worker options.
:type options: dict
"""
self.log.debug(
"NodeControllerSession.start_container(id = {id}, options = {options})",
id=id,
options=options)
return self._start_native_worker('container',
id,
options,
details=details)
def start_websocket_testee(self, id, options=None, details=None):
"""
Start a new websocket-testee worker: a Crossbar.io native worker process
that runs a plain echo'ing WebSocket server.
:param id: The worker ID to start this router with.
:type id: str
:param options: The worker options.
:type options: dict
"""
self.log.debug(
"NodeControllerSession.start_websocket_testee({id}, options={options})",
id=id,
options=options)
return self._start_native_worker('websocket-testee',
id,
options,
details=details)
def _start_native_worker(self, wtype, id, options=None, details=None):
assert (wtype in ['router', 'container', 'websocket-testee'])
# prohibit starting a worker twice
#
if id in self._workers:
emsg = "Could not start worker: a worker with ID '{}' is already running (or starting)".format(
id)
self.log.error(emsg)
raise ApplicationError(u'crossbar.error.worker_already_running',
emsg)
# check worker options
#
options = options or {}
try:
if wtype == 'router':
checkconfig.check_router_options(options)
elif wtype == 'container':
checkconfig.check_container_options(options)
elif wtype == 'websocket-testee':
checkconfig.check_websocket_testee_options(options)
else:
raise Exception("logic error")
except Exception as e:
emsg = "Could not start native worker: invalid configuration ({})".format(
e)
self.log.error(emsg)
raise ApplicationError(u'crossbar.error.invalid_configuration',
emsg)
# allow override Python executable from options
#
if 'python' in options:
exe = options['python']
# the executable must be an absolute path, e.g. /home/oberstet/pypy-2.2.1-linux64/bin/pypy
#
if not os.path.isabs(exe):
emsg = "Invalid worker configuration: python executable '{}' must be an absolute path".format(
exe)
self.log.error(emsg)
raise ApplicationError(u'crossbar.error.invalid_configuration',
emsg)
# of course the path must exist and actually be executable
#
if not (os.path.isfile(exe) and os.access(exe, os.X_OK)):
emsg = "Invalid worker configuration: python executable '{}' does not exist or isn't an executable".format(
exe)
self.log.error(emsg)
raise ApplicationError(u'crossbar.error.invalid_configuration',
emsg)
else:
exe = sys.executable
# all native workers (routers and containers for now) start from the same script
#
filename = os.path.abspath(
os.path.join(crossbar.__file__, "..", "worker", "process.py"))
# assemble command line for forking the worker
#
args = [exe, "-u", filename]
args.extend(["--cbdir", self._node._cbdir])
args.extend(["--node", str(self._node_id)])
args.extend(["--worker", str(id)])
args.extend(["--realm", self._realm])
args.extend(["--type", wtype])
args.extend(["--loglevel", _loglevel])
# allow override worker process title from options
#
if options.get('title', None):
args.extend(['--title', options['title']])
# forward explicit reactor selection
#
if 'reactor' in options and sys.platform in options['reactor']:
args.extend(['--reactor', options['reactor'][sys.platform]])
# FIXME
# elif self._node.options.reactor:
# args.extend(['--reactor', self._node.options.reactor])
# create worker process environment
#
worker_env = create_process_env(options)
# We need to use the same PYTHONPATH we were started with, so we can
# find the Crossbar we're working with -- it may not be the same as the
# one on the default path
worker_env["PYTHONPATH"] = os.pathsep.join(sys.path)
# log name of worker
#
worker_logname = {
'router': 'Router',
'container': 'Container',
'websocket-testee': 'WebSocketTestee'
}.get(wtype, 'Worker')
# topic URIs used (later)
#
if wtype == 'router':
starting_topic = 'crossbar.node.{}.on_router_starting'.format(
self._node_id)
started_topic = 'crossbar.node.{}.on_router_started'.format(
self._node_id)
elif wtype == 'container':
starting_topic = 'crossbar.node.{}.on_container_starting'.format(
self._node_id)
started_topic = 'crossbar.node.{}.on_container_started'.format(
self._node_id)
elif wtype == 'websocket-testee':
starting_topic = 'crossbar.node.{}.on_websocket_testee_starting'.format(
self._node_id)
started_topic = 'crossbar.node.{}.on_websocket_testee_started'.format(
self._node_id)
else:
raise Exception("logic error")
# add worker tracking instance to the worker map ..
#
if wtype == 'router':
worker = RouterWorkerProcess(self,
id,
details.caller,
keeplog=options.get(
'traceback', None))
elif wtype == 'container':
worker = ContainerWorkerProcess(self,
id,
details.caller,
keeplog=options.get(
'traceback', None))
elif wtype == 'websocket-testee':
worker = WebSocketTesteeWorkerProcess(self,
id,
details.caller,
keeplog=options.get(
'traceback', None))
else:
raise Exception("logic error")
self._workers[id] = worker
# create a (custom) process endpoint.
#
if platform.isWindows():
childFDs = None # Use the default Twisted ones
else:
# The communication between controller and container workers is
# using WAMP running over 2 pipes.
# For controller->container traffic this runs over FD 0 (`stdin`)
# and for the container->controller traffic, this runs over FD 3.
#
# Note: We use FD 3, not FD 1 (`stdout`) or FD 2 (`stderr`) for
# container->controller traffic, so that components running in the
# container which happen to write to `stdout` or `stderr` do not
# interfere with the container-controller communication.
childFDs = {0: "w", 1: "r", 2: "r", 3: "r"}
ep = WorkerProcessEndpoint(self._node._reactor,
exe,
args,
env=worker_env,
worker=worker,
childFDs=childFDs)
# ready handling
#
def on_ready_success(id):
self.log.info("{worker} with ID '{id}' and PID {pid} started",
worker=worker_logname,
id=worker.id,
pid=worker.pid)
self._node._reactor.addSystemEventTrigger(
'before',
'shutdown',
self._cleanup_worker,
self._node._reactor,
worker,
)
worker.status = 'started'
worker.started = datetime.utcnow()
started_info = {
'id': worker.id,
'status': worker.status,
'started': utcstr(worker.started),
'who': worker.who
}
# FIXME: make start of stats printer dependent on log level ..
worker.log_stats(5.)
self.publish(started_topic,
started_info,
options=PublishOptions(exclude=[details.caller]))
return started_info
def on_ready_error(err):
del self._workers[worker.id]
emsg = 'Failed to start native worker: {}'.format(err.value)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.cannot_start", emsg,
worker.getlog())
worker.ready.addCallbacks(on_ready_success, on_ready_error)
def on_exit_success(_):
self.log.info("Node worker {} ended successfully".format(
worker.id))
worker.log_stats(0)
del self._workers[worker.id]
return True
def on_exit_error(err):
self.log.info("Node worker {} ended with error ({})".format(
worker.id, err))
worker.log_stats(0)
del self._workers[worker.id]
return False
def check_for_shutdown(was_successful):
shutdown = False
# automatically shutdown node whenever a worker ended (successfully, or with error)
#
if checkconfig.NODE_SHUTDOWN_ON_WORKER_EXIT in self._node._node_shutdown_triggers:
self.log.info(
"Node worker ended, and trigger '{}' active".format(
checkconfig.NODE_SHUTDOWN_ON_WORKER_EXIT))
shutdown = True
# automatically shutdown node when worker ended with error
#
if not was_successful and checkconfig.NODE_SHUTDOWN_ON_WORKER_EXIT_WITH_ERROR in self._node._node_shutdown_triggers:
self.log.info(
"Node worker ended with error, and trigger '{}' active".
format(
checkconfig.NODE_SHUTDOWN_ON_WORKER_EXIT_WITH_ERROR))
shutdown = True
# automatically shutdown node when no more workers are left
#
if len(
self._workers
) == 0 and checkconfig.NODE_SHUTDOWN_ON_LAST_WORKER_EXIT in self._node._node_shutdown_triggers:
self.log.info(
"No more node workers running, and trigger '{}' active".
format(checkconfig.NODE_SHUTDOWN_ON_LAST_WORKER_EXIT))
shutdown = True
# initiate shutdown (but only if we are not already shutting down)
#
if shutdown:
if not self._shutdown_requested:
self.log.info("Node shutting down ..")
self.shutdown()
else:
# ignore: shutdown already initiated ..
self.log.info("Node is already shutting down.")
else:
self.log.info(
"Node will continue to run (node shutdown triggers active: {})"
.format(self._node._node_shutdown_triggers))
d_on_exit = worker.exit.addCallbacks(on_exit_success, on_exit_error)
d_on_exit.addBoth(check_for_shutdown)
# create a transport factory for talking WAMP to the native worker
#
transport_factory = create_native_worker_client_factory(
self._node._router_session_factory, worker.ready, worker.exit)
transport_factory.noisy = False
self._workers[id].factory = transport_factory
# now (immediately before actually forking) signal the starting of the worker
#
starting_info = {
'id': id,
'status': worker.status,
'created': utcstr(worker.created),
'who': worker.who
}
# the caller gets a progressive result ..
if details.progress:
details.progress(starting_info)
# .. while all others get an event
self.publish(starting_topic,
starting_info,
options=PublishOptions(exclude=[details.caller]))
# now actually fork the worker ..
#
self.log.info("Starting {worker} with ID '{id}'...",
worker=worker_logname,
id=id)
self.log.debug("{worker} '{id}' command line is '{cmdline}'",
worker=worker_logname,
id=id,
cmdline=' '.join(args))
d = ep.connect(transport_factory)
def on_connect_success(proto):
# this seems to be called immediately when the child process
# has been forked. even if it then immediately fails because
# e.g. the executable doesn't even exist. in other words,
# I'm not sure under what conditions the deferred will errback ..
pid = proto.transport.pid
self.log.debug("Native worker process connected with PID {pid}",
pid=pid)
# note the PID of the worker
worker.pid = pid
# proto is an instance of NativeWorkerClientProtocol
worker.proto = proto
worker.status = 'connected'
worker.connected = datetime.utcnow()
def on_connect_error(err):
# not sure when this errback is triggered at all ..
self.log.error(
"Interal error: connection to forked native worker failed ({err})",
err=err)
# in any case, forward the error ..
worker.ready.errback(err)
d.addCallbacks(on_connect_success, on_connect_error)
return worker.ready
@staticmethod
def _cleanup_worker(reactor, worker):
"""
This is called during reactor shutdown and ensures we wait for our
subprocesses to shut down nicely.
"""
log = make_logger()
try:
log.info("sending TERM to subprocess {pid}", pid=worker.pid)
worker.proto.transport.signalProcess('TERM')
# wait for the subprocess to shutdown; could add a timeout
# after which we send a KILL maybe?
d = Deferred()
def protocol_closed(_):
log.debug("{pid} exited", pid=worker.pid)
d.callback(None)
# await worker's timely demise
worker.exit.addCallback(protocol_closed)
def timeout(tried):
if d.called:
return
log.info("waiting for {pid} to exit...", pid=worker.pid)
reactor.callLater(1, timeout, tried + 1)
if tried > 20: # or just wait forever?
log.info("Sending SIGKILL to {pid}", pid=worker.pid)
try:
worker.proto.transport.signalProcess('KILL')
except ProcessExitedAlready:
pass # ignore; it's already dead
d.callback(None) # or recurse more?
timeout(0)
return d
except ProcessExitedAlready:
pass # ignore; it's already dead
def stop_router(self, id, kill=False, details=None):
"""
Stops a currently running router worker.
:param id: The ID of the router worker to stop.
:type id: str
:param kill: If `True`, kill the process. Otherwise, gracefully
shut down the worker.
:type kill: bool
:returns: Stopping information from the worker.
:rtype: dict
"""
self.log.debug("NodeControllerSession.stop_router({id}, kill={kill})",
id=id,
kill=kill)
return self._stop_native_worker('router', id, kill, details=details)
def stop_container(self, id, kill=False, details=None):
"""
Stops a currently running container worker.
:param id: The ID of the container worker to stop.
:type id: str
:param kill: If `True`, kill the process. Otherwise, gracefully
shut down the worker.
:type kill: bool
:returns: Stopping information from the worker.
:rtype: dict
"""
self.log.debug(
"NodeControllerSession.stop_container({id}, kill={kill})",
id=id,
kill=kill)
return self._stop_native_worker('container', id, kill, details=details)
def stop_websocket_testee(self, id, kill=False, details=None):
"""
Stops a currently running websocket-testee worker.
:param id: The ID of the worker to stop.
:type id: str
:param kill: If `True`, kill the process. Otherwise, gracefully
shut down the worker.
:type kill: bool
:returns: Stopping information from the worker.
:rtype: dict
"""
self.log.debug(
"NodeControllerSession.stop_websocket_testee({id}, kill={kill})",
id=id,
kill=kill)
return self._stop_native_worker('websocket-testee',
id,
kill,
details=details)
@inlineCallbacks
def _stop_native_worker(self, wtype, id, kill, details=None):
assert (wtype in ['router', 'container', 'websocket-testee'])
if id not in self._workers or self._workers[id].TYPE != wtype:
emsg = "Could not stop native worker: no {} worker with ID '{}' currently running".format(
wtype, id)
raise ApplicationError(u'crossbar.error.worker_not_running', emsg)
worker = self._workers[id]
if worker.status != 'started':
emsg = "Could not stop native worker: worker with ID '{}' is not in status 'started', but status: '{}')".format(
id, worker.status)
raise ApplicationError(u'crossbar.error.worker_not_running', emsg)
stop_info = {
'id': worker.id,
'type': wtype,
'kill': kill,
'who': details.caller if details else None,
'when': utcnow()
}
# publish management API event
#
yield self.publish(
'crossbar.node.{}.worker.{}.on_stop_requested'.format(
self._node_id, worker.id),
stop_info,
options=PublishOptions(
exclude=[details.caller] if details else None,
acknowledge=True))
# send SIGKILL or SIGTERM to worker
#
if kill:
self.log.info("Killing {wtype} worker with ID '{id}'",
wtype=wtype,
id=id)
self._workers[id].proto.transport.signalProcess("KILL")
else:
self.log.info("Stopping {wtype} worker with ID '{id}'",
wtype=wtype,
id=id)
self._workers[id].factory.stopFactory()
self._workers[id].proto.transport.signalProcess('TERM')
returnValue(stop_info)
def start_guest(self, id, config, details=None):
"""
Start a new guest process on this node.
:param config: The guest process configuration.
:type config: obj
:returns: int -- The PID of the new process.
"""
# prohibit starting a worker twice
#
if id in self._workers:
emsg = "Could not start worker: a worker with ID '{}' is already running (or starting)".format(
id)
self.log.error(emsg)
raise ApplicationError(u'crossbar.error.worker_already_running',
emsg)
try:
checkconfig.check_guest(config)
except Exception as e:
raise ApplicationError(
u'crossbar.error.invalid_configuration',
'invalid guest worker configuration: {}'.format(e))
options = config.get('options', {})
# guest process working directory
#
workdir = self._node._cbdir
if 'workdir' in options:
workdir = os.path.join(workdir, options['workdir'])
workdir = os.path.abspath(workdir)
# guest process executable and command line arguments
#
# first try to configure the fully qualified path for the guest
# executable by joining workdir and configured exectuable ..
exe = os.path.abspath(os.path.join(workdir, config['executable']))
if check_executable(exe):
self.log.info(
"Using guest worker executable '{exe}' (executable path taken from configuration)",
exe=exe)
else:
# try to detect the fully qualified path for the guest
# executable by doing a "which" on the configured executable name
exe = shutil.which(config['executable'])
if exe is not None and check_executable(exe):
self.log.info(
"Using guest worker executable '{exe}' (executable path detected from environment)",
exe=exe)
else:
emsg = "Could not start worker: could not find and executable for '{}'".format(
config['executable'])
self.log.error(emsg)
raise ApplicationError(u'crossbar.error.invalid_configuration',
emsg)
# guest process command line arguments
#
args = [exe]
args.extend(config.get('arguments', []))
# guest process environment
#
worker_env = create_process_env(options)
# log name of worker
#
worker_logname = 'Guest'
# topic URIs used (later)
#
starting_topic = 'crossbar.node.{}.on_guest_starting'.format(
self._node_id)
started_topic = 'crossbar.node.{}.on_guest_started'.format(
self._node_id)
# add worker tracking instance to the worker map ..
#
worker = GuestWorkerProcess(self,
id,
details.caller,
keeplog=options.get('traceback', None))
self._workers[id] = worker
# create a (custom) process endpoint
#
ep = WorkerProcessEndpoint(self._node._reactor,
exe,
args,
path=workdir,
env=worker_env,
worker=worker)
# ready handling
#
def on_ready_success(proto):
worker.pid = proto.transport.pid
worker.status = 'started'
worker.started = datetime.utcnow()
self.log.info("{worker} with ID '{id}' and PID {pid} started",
worker=worker_logname,
id=worker.id,
pid=worker.pid)
self._node._reactor.addSystemEventTrigger(
'before',
'shutdown',
self._cleanup_worker,
self._node._reactor,
worker,
)
# directory watcher
#
if 'watch' in options:
if HAS_FSNOTIFY:
# assemble list of watched directories
watched_dirs = []
for d in options['watch'].get('directories', []):
watched_dirs.append(
os.path.abspath(os.path.join(self._node._cbdir,
d)))
worker.watch_timeout = options['watch'].get('timeout', 1)
# create a directory watcher
worker.watcher = DirWatcher(dirs=watched_dirs,
notify_once=True)
# make sure to stop the background thread running inside the
# watcher upon Twisted being shut down
def on_shutdown():
worker.watcher.stop()
self._node._reactor.addSystemEventTrigger(
'before', 'shutdown', on_shutdown)
# this handler will get fired by the watcher upon detecting an FS event
def on_fsevent(evt):
worker.watcher.stop()
proto.signal('TERM')
if options['watch'].get('action', None) == 'restart':
self.log.info("Restarting guest ..")
# Add a timeout large enough (perhaps add a config option later)
self._node._reactor.callLater(
worker.watch_timeout, self.start_guest, id,
config, details)
# Shut the worker down, after the restart event is scheduled
worker.stop()
# now run the watcher on a background thread
deferToThread(worker.watcher.loop, on_fsevent)
else:
self.log.warn(
"Warning: cannot watch directory for changes - feature DirWatcher unavailable"
)
# assemble guest worker startup information
#
started_info = {
'id': worker.id,
'status': worker.status,
'started': utcstr(worker.started),
'who': worker.who
}
self.publish(started_topic,
started_info,
options=PublishOptions(exclude=[details.caller]))
return started_info
def on_ready_error(err):
del self._workers[worker.id]
emsg = 'Failed to start guest worker: {}'.format(err.value)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.cannot_start", emsg,
ep.getlog())
worker.ready.addCallbacks(on_ready_success, on_ready_error)
def on_exit_success(res):
self.log.info("Guest {id} exited with success", id=worker.id)
del self._workers[worker.id]
def on_exit_error(err):
self.log.error("Guest {id} exited with error {err.value}",
id=worker.id,
err=err)
del self._workers[worker.id]
worker.exit.addCallbacks(on_exit_success, on_exit_error)
# create a transport factory for talking WAMP to the native worker
#
transport_factory = create_guest_worker_client_factory(
config, worker.ready, worker.exit)
transport_factory.noisy = False
self._workers[id].factory = transport_factory
# now (immediately before actually forking) signal the starting of the worker
#
starting_info = {
'id': id,
'status': worker.status,
'created': utcstr(worker.created),
'who': worker.who
}
# the caller gets a progressive result ..
if details.progress:
details.progress(starting_info)
# .. while all others get an event
self.publish(starting_topic,
starting_info,
options=PublishOptions(exclude=[details.caller]))
# now actually fork the worker ..
#
self.log.info("Starting {worker} with ID '{id}'...",
worker=worker_logname,
id=id)
self.log.debug("{worker} '{id}' using command line '{cli}'...",
worker=worker_logname,
id=id,
cli=' '.join(args))
d = ep.connect(transport_factory)
def on_connect_success(proto):
# this seems to be called immediately when the child process
# has been forked. even if it then immediately fails because
# e.g. the executable doesn't even exist. in other words,
# I'm not sure under what conditions the deferred will
# errback - probably only if the forking of a new process fails
# at OS level due to out of memory conditions or such.
pid = proto.transport.pid
self.log.debug("Guest worker process connected with PID {pid}",
pid=pid)
worker.pid = pid
# proto is an instance of GuestWorkerClientProtocol
worker.proto = proto
worker.status = 'connected'
worker.connected = datetime.utcnow()
def on_connect_error(err):
# not sure when this errback is triggered at all .. see above.
self.log.error(
"Internal error: connection to forked guest worker failed ({})"
.format(err))
# in any case, forward the error ..
worker.ready.errback(err)
d.addCallbacks(on_connect_success, on_connect_error)
return worker.ready
def stop_guest(self, id, kill=False, details=None):
"""
Stops a currently running guest worker.
:param id: The ID of the guest worker to stop.
:type id: str
"""
self.log.debug("NodeControllerSession.stop_guest({id}, kill={kill})",
id=id,
kill=kill)
if id not in self._workers or self._workers[id].worker_type != 'guest':
emsg = "Could not stop guest worker: no guest worker with ID '{}' currently running".format(
id)
raise ApplicationError(u'crossbar.error.worker_not_running', emsg)
try:
if kill:
self._workers[id].proto.transport.signalProcess("KILL")
else:
self._workers[id].proto.transport.loseConnection()
except Exception as e:
emsg = "Could not stop guest worker with ID '{}': {}".format(id, e)
raise ApplicationError(u'crossbar.error.stop_worker_failed', emsg)
else:
del self._workers[id]
def run_command_start(options, reactor=None):
"""
Subcommand "crossbar start".
"""
assert reactor
# do not allow to run more than one Crossbar.io instance
# from the same Crossbar.io node directory
#
pid_data = check_is_running(options.cbdir)
if pid_data:
print(
"Crossbar.io is already running from node directory {} (PID {}).".
format(options.cbdir, pid_data['pid']))
sys.exit(1)
else:
fp = os.path.join(options.cbdir, _PID_FILENAME)
with open(fp, 'wb') as fd:
argv = options.argv
options_dump = vars(options)
pid_data = {
'pid': os.getpid(),
'argv': argv,
'options': {
x: y
for x, y in options_dump.items()
if x not in ["func", "argv"]
}
}
fd.write("{}\n".format(
json.dumps(pid_data,
sort_keys=False,
indent=4,
separators=(', ', ': '),
ensure_ascii=False)).encode('utf8'))
# remove node PID file when reactor exits
#
def remove_pid_file():
fp = os.path.join(options.cbdir, _PID_FILENAME)
if os.path.isfile(fp):
os.remove(fp)
reactor.addSystemEventTrigger('after', 'shutdown', remove_pid_file)
log = make_logger()
# possibly generate new node key
#
from crossbar.controller.node import maybe_generate_key
pubkey = maybe_generate_key(log, options.cbdir)
# Print the banner.
for line in BANNER.splitlines():
log.info(click.style(("{:>40}").format(line), fg='yellow', bold=True))
# bannerFormat = "{:<18} {:<24}"
bannerFormat = " {} {}"
log.info(
bannerFormat.format(
"Crossbar.io Version:",
click.style(crossbar.__version__, fg='yellow', bold=True)))
if pubkey:
log.info(
bannerFormat.format("Node Public Key:",
click.style(pubkey, fg='yellow', bold=True)))
log.info()
log.info("Running from node directory '{cbdir}'", cbdir=options.cbdir)
from twisted.python.reflect import qual
log.info("Controller process starting ({python}-{reactor}) ..",
python=platform.python_implementation(),
reactor=qual(reactor.__class__).split('.')[-1])
from crossbar.controller.node import Node
from crossbar.common.checkconfig import InvalidConfigException
# represents the running Crossbar.io node
#
node = Node(options.cbdir, reactor=reactor)
# check and load the node configuration
#
try:
if options.config:
# load node config from file
node.load(options.config)
elif options.cdc:
# load built-in CDC config
node.load()
else:
# no config file, and not running CDC mode
raise Exception(
"Neither a node config was found, nor CDC mode is active.")
except InvalidConfigException as e:
log.error("Invalid node configuration")
log.error("{e!s}", e=e)
sys.exit(1)
except:
raise
# now actually start the node ..
#
def start_crossbar():
d = node.start(cdc_mode=options.cdc)
def on_error(err):
log.error("{e!s}", e=err.value)
log.error("Could not start node")
if reactor.running:
reactor.stop()
d.addErrback(on_error)
reactor.callWhenRunning(start_crossbar)
# enter event loop
#
try:
reactor.run()
except Exception:
log.failure("Could not start reactor - {log_failure.value}")
def run_command_start(options):
"""
Subcommand "crossbar start".
"""
# do not allow to run more than one Crossbar.io instance
# from the same Crossbar.io node directory
#
pid_data = check_is_running(options.cbdir)
if pid_data:
print(
"Crossbar.io is already running from node directory {} (PID {}).".
format(options.cbdir, pid_data['pid']))
sys.exit(1)
else:
fp = os.path.join(options.cbdir, _PID_FILENAME)
with open(fp, 'w') as fd:
argv = options.argv
options_dump = vars(options)
del options_dump['func']
del options_dump['argv']
pid_data = {
'pid': os.getpid(),
'argv': argv,
'options': options_dump
}
fd.write("{}\n".format(
json.dumps(pid_data,
sort_keys=False,
indent=3,
separators=(',', ': '))))
# we use an Autobahn utility to import the "best" available Twisted reactor
#
reactor = install_reactor(options.reactor, options.debug)
# remove node PID file when reactor exits
#
def remove_pid_file():
fp = os.path.join(options.cbdir, _PID_FILENAME)
if os.path.isfile(fp):
os.remove(fp)
reactor.addSystemEventTrigger('after', 'shutdown', remove_pid_file)
# start Twisted logging
#
from crossbar._logging import log_publisher, make_logger
from crossbar._logging import start_logging, set_global_log_level
set_global_log_level(options.loglevel)
log = make_logger()
if options.logdir:
# We want to log to a file
from crossbar._logging import make_legacy_daily_logfile_observer
log_publisher.addObserver(
make_legacy_daily_logfile_observer(options.logdir,
options.loglevel))
else:
# We want to log to stdout/stderr.
from crossbar._logging import make_stdout_observer
from crossbar._logging import make_stderr_observer
if options.loglevel == "none":
# Do no logging!
pass
elif options.loglevel == "error":
# Error: Only print errors to stderr.
log_publisher.addObserver(
make_stdout_observer(show_source=False,
format=options.logformat))
log_publisher.addObserver(
make_stderr_observer(show_source=False,
format=options.logformat))
elif options.loglevel == "warn":
# Print warnings+ to stderr.
log_publisher.addObserver(
make_stdout_observer(show_source=False,
format=options.logformat))
log_publisher.addObserver(
make_stderr_observer(show_source=False,
format=options.logformat))
elif options.loglevel == "info":
# Print info to stdout, warn+ to stderr
log_publisher.addObserver(
make_stdout_observer(show_source=False,
format=options.logformat))
log_publisher.addObserver(
make_stderr_observer(show_source=False,
format=options.logformat))
elif options.loglevel == "debug":
# Print debug+info to stdout, warn+ to stderr, with the class
# source
log_publisher.addObserver(
make_stdout_observer(show_source=True,
format=options.logformat))
log_publisher.addObserver(
make_stderr_observer(show_source=True,
format=options.logformat))
elif options.loglevel == "trace":
# Print trace+, with the class source
log_publisher.addObserver(
make_stdout_observer(show_source=True,
format=options.logformat,
trace=True))
log_publisher.addObserver(
make_stderr_observer(show_source=True,
format=options.logformat))
else:
assert False, "Shouldn't ever get here."
# Actually start the logger.
start_logging()
for line in BANNER.splitlines():
log.info(click.style(("{:>40}").format(line), fg='yellow', bold=True))
bannerFormat = "{:>12} {:<24}"
log.info(
bannerFormat.format(
"Version:",
click.style(crossbar.__version__, fg='yellow', bold=True)))
# log.info(bannerFormat.format("Python:", click.style(platform.python_implementation(), fg='yellow', bold=True)))
# log.info(bannerFormat.format("Reactor:", click.style(qual(reactor.__class__).split('.')[-1], fg='yellow', bold=True)))
log.info(
bannerFormat.format("Started:",
click.style(utcnow(), fg='yellow', bold=True)))
log.info()
log.info("Starting from node directory {}".format(options.cbdir))
# create and start Crossbar.io node
#
def start_crossbar():
from crossbar.controller.node import Node
node = Node(reactor, options)
d = node.start()
def on_error(err):
log.error("Could not start node: {error}", error=err.value)
if reactor.running:
reactor.stop()
d.addErrback(on_error)
reactor.callWhenRunning(start_crossbar)
try:
log.info("Entering reactor event loop...")
reactor.run()
except Exception:
log.failure("Could not start reactor: {log_failure.value}")
class NativeProcessSession(ApplicationSession):
"""
A native Crossbar.io process (currently: controller, router or container).
"""
log = make_logger()
def __init__(self, config=None, reactor=None):
if not reactor:
from twisted.internet import reactor
self._reactor = reactor
self._reactor = reactor
super(ApplicationSession, self).__init__(config=config)
def onConnect(self, do_join=True):
"""
"""
if not hasattr(self, 'cbdir'):
self.cbdir = self.config.extra.cbdir
if not hasattr(self, '_uri_prefix'):
self._uri_prefix = 'crossbar.node.{}'.format(self.config.extra.node)
self._started = datetime.utcnow()
# see: BaseSession
self.include_traceback = False
self.debug_app = False
self._manhole_service = None
if _HAS_PSUTIL:
self._pinfo = ProcessInfo()
self._pinfo_monitor = None
self._pinfo_monitor_seq = 0
else:
self._pinfo = None
self._pinfo_monitor = None
self._pinfo_monitor_seq = None
self.log.info("Process utilities not available")
self._connections = {}
if do_join:
self.join(self.config.realm)
@inlineCallbacks
def onJoin(self, details):
"""
Called when process has joined the node's management realm.
"""
procs = [
'start_manhole',
'stop_manhole',
'get_manhole',
'start_connection',
'stop_connection',
'get_connections',
'trigger_gc',
'utcnow',
'started',
'uptime',
'get_process_info',
'get_process_stats',
'set_process_stats_monitoring'
]
dl = []
for proc in procs:
uri = '{}.{}'.format(self._uri_prefix, proc)
self.log.debug("Registering procedure '{uri}'", uri=uri)
dl.append(self.register(getattr(self, proc), uri, options=RegisterOptions(details_arg='details')))
regs = yield DeferredList(dl)
self.log.debug("Registered {len_reg} procedures", len_reg=len(regs))
@inlineCallbacks
def start_connection(self, id, config, details=None):
"""
Starts a connection in this process.
:param id: The ID for the started connection.
:type id: unicode
:param config: Connection configuration.
:type config: dict
:param details: Caller details.
:type details: instance of :class:`autobahn.wamp.types.CallDetails`
:returns dict -- The connection.
"""
self.log.debug("start_connection: id={id}, config={config}", id=id, config=config)
# prohibit starting a component twice
#
if id in self._connections:
emsg = "cannot start connection: a connection with id={} is already started".format(id)
self.log.warn(emsg)
raise ApplicationError(u"crossbar.error.invalid_configuration", emsg)
# check configuration
#
try:
checkconfig.check_connection(config)
except Exception as e:
emsg = "invalid connection configuration ({})".format(e)
self.log.warn(emsg)
raise ApplicationError(u"crossbar.error.invalid_configuration", emsg)
else:
self.log.info("Starting {} in process.".format(config['type']))
if config['type'] == u'postgresql.connection':
if _HAS_POSTGRESQL:
connection = PostgreSQLConnection(id, config)
else:
emsg = "unable to start connection - required PostgreSQL driver package not installed"
self.log.warn(emsg)
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
else:
# should not arrive here
raise Exception("logic error")
self._connections[id] = connection
try:
yield connection.start()
self.log.info("Connection {connection_type} started '{connection_id}'", connection_id=id, connection_type=config['type'])
except Exception as e:
del self._connections[id]
raise
state = connection.marshal()
self.publish(u'crossbar.node.process.on_connection_start', state)
returnValue(state)
@inlineCallbacks
def stop_connection(self, id, details=None):
"""
Stop a connection currently running within this process.
:param id: The ID of the connection to stop.
:type id: unicode
:param details: Caller details.
:type details: instance of :class:`autobahn.wamp.types.CallDetails`
:returns dict -- A dict with component start information.
"""
self.log.debug("stop_connection: id={id}", id=id)
if id not in self._connections:
raise ApplicationError(u'crossbar.error.no_such_object', 'no connection with ID {} running in this process'.format(id))
connection = self._connections[id]
try:
yield connection.stop()
except Exception as e:
self.log.warn('could not stop connection {id}: {error}', error=e)
raise
del self._connections[id]
state = connection.marshal()
self.publish(u'crossbar.node.process.on_connection_stop', state)
returnValue(state)
def get_connections(self, details=None):
"""
Get connections currently running within this processs.
:param details: Caller details.
:type details: instance of :class:`autobahn.wamp.types.CallDetails`
:returns list -- List of connections.
"""
self.log.debug("get_connections")
res = []
for c in self._connections.values():
res.append(c.marshal())
return res
def get_process_info(self, details=None):
"""
Get process information (open files, sockets, ...).
:returns: dict -- Dictionary with process information.
"""
self.log.debug("{cls}.get_process_info",
cls=self.__class__.__name__)
if self._pinfo:
return self._pinfo.get_info()
else:
emsg = "Could not retrieve process statistics: required packages not installed"
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
def get_process_stats(self, details=None):
"""
Get process statistics (CPU, memory, I/O).
:returns: dict -- Dictionary with process statistics.
"""
self.log.debug("{cls}.get_process_stats", cls=self.__class__.__name__)
if self._pinfo:
return self._pinfo.get_stats()
else:
emsg = "Could not retrieve process statistics: required packages not installed"
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
def set_process_stats_monitoring(self, interval, details=None):
"""
Enable/disable periodic publication of process statistics.
:param interval: The monitoring interval in seconds. Set to 0 to disable monitoring.
:type interval: float
"""
self.log.debug("{cls}.set_process_stats_monitoring(interval = {interval})",
cls=self.__class__.__name__, interval=interval)
if self._pinfo:
stats_monitor_set_topic = '{}.on_process_stats_monitoring_set'.format(self._uri_prefix)
# stop and remove any existing monitor
if self._pinfo_monitor:
self._pinfo_monitor.stop()
self._pinfo_monitor = None
self.publish(stats_monitor_set_topic, 0, options=PublishOptions(exclude=details.caller))
# possibly start a new monitor
if interval > 0:
stats_topic = '{}.on_process_stats'.format(self._uri_prefix)
def publish_stats():
stats = self._pinfo.get_stats()
self._pinfo_monitor_seq += 1
stats[u'seq'] = self._pinfo_monitor_seq
self.publish(stats_topic, stats)
self._pinfo_monitor = LoopingCall(publish_stats)
self._pinfo_monitor.start(interval)
self.publish(stats_monitor_set_topic, interval, options=PublishOptions(exclude=details.caller))
else:
emsg = "Cannot setup process statistics monitor: required packages not installed"
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
def trigger_gc(self, details=None):
"""
Manually trigger a garbage collection in this native process.
This procedure is registered under
``crossbar.node.<node_id>.worker.<worker_id>.trigger_gc``
for native workers and under
``crossbar.node.<node_id>.controller.trigger_gc``
for node controllers.
The procedure will publish an event when the garabage collection has finished to
``crossbar.node.<node_id>.worker.<worker_id>.on_gc_finished``
for native workers and
``crossbar.node.<node_id>.controller.on_gc_finished``
for node controllers:
.. code-block:: javascript
{
"requester": {
"session_id": 982734923,
"auth_id": "bob",
"auth_role": "admin"
},
"duration": 190
}
.. note:: The caller of this procedure will NOT receive the event.
:returns: Time (wall clock) consumed for garbage collection in ms.
:rtype: int
"""
self.log.debug("{cls}.trigger_gc", cls=self.__class__.__name__)
started = rtime()
# now trigger GC .. this is blocking!
gc.collect()
duration = int(round(1000. * (rtime() - started)))
on_gc_finished = u'{}.on_gc_finished'.format(self._uri_prefix)
self.publish(
on_gc_finished,
{
u'requester': {
u'session_id': details.caller,
# FIXME:
u'auth_id': None,
u'auth_role': None
},
u'duration': duration
},
options=PublishOptions(exclude=details.caller)
)
return duration
@inlineCallbacks
def start_manhole(self, config, details=None):
"""
Start a Manhole service within this process.
**Usage:**
This procedure is registered under
* ``crossbar.node.<node_id>.worker.<worker_id>.start_manhole`` - for native workers
* ``crossbar.node.<node_id>.controller.start_manhole`` - for node controllers
The procedure takes a Manhole service configuration which defines
a listening endpoint for the service and a list of users including
passwords, e.g.
.. code-block:: javascript
{
"endpoint": {
"type": "tcp",
"port": 6022
},
"users": [
{
"user": "******",
"password": "******"
}
]
}
**Errors:**
The procedure may raise the following errors:
* ``crossbar.error.invalid_configuration`` - the provided configuration is invalid
* ``crossbar.error.already_started`` - the Manhole service is already running (or starting)
* ``crossbar.error.feature_unavailable`` - the required support packages are not installed
**Events:**
The procedure will publish an event when the service **is starting** to
* ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_starting`` - for native workers
* ``crossbar.node.<node_id>.controller.on_manhole_starting`` - for node controllers
and publish an event when the service **has started** to
* ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_started`` - for native workers
* ``crossbar.node.<node_id>.controller.on_manhole_started`` - for node controllers
:param config: Manhole service configuration.
:type config: dict
"""
self.log.debug("{cls}.start_manhole(config = {config})",
cls=self.__class__.__name__, config=config)
if not _HAS_MANHOLE:
emsg = "Could not start manhole: required packages are missing ({})".format(_MANHOLE_MISSING_REASON)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
if self._manhole_service:
emsg = "Could not start manhole - already running (or starting)"
self.log.warn(emsg)
raise ApplicationError(u"crossbar.error.already_started", emsg)
try:
checkconfig.check_manhole(config)
except Exception as e:
emsg = "Could not start manhole: invalid configuration ({})".format(e)
self.log.error(emsg)
raise ApplicationError(u'crossbar.error.invalid_configuration', emsg)
# setup user authentication
#
checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()
for user in config['users']:
checker.addUser(user['user'], user['password'])
# setup manhole namespace
#
namespace = {'session': self}
class PatchedTerminalSession(TerminalSession):
# get rid of
# exceptions.AttributeError: TerminalSession instance has no attribute 'windowChanged'
def windowChanged(self, winSize):
pass
rlm = TerminalRealm()
rlm.sessionFactory = PatchedTerminalSession # monkey patch
rlm.chainedProtocolFactory.protocolFactory = lambda _: ColoredManhole(namespace)
ptl = portal.Portal(rlm, [checker])
factory = ConchFactory(ptl)
factory.noisy = False
self._manhole_service = ManholeService(config, details.caller)
starting_topic = '{}.on_manhole_starting'.format(self._uri_prefix)
starting_info = self._manhole_service.marshal()
# the caller gets a progressive result ..
if details.progress:
details.progress(starting_info)
# .. while all others get an event
self.publish(starting_topic, starting_info, options=PublishOptions(exclude=details.caller))
try:
self._manhole_service.port = yield create_listening_port_from_config(config['endpoint'],
self.cbdir,
factory,
self._reactor,
self.log)
except Exception as e:
self._manhole_service = None
emsg = "Manhole service endpoint cannot listen: {}".format(e)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.cannot_listen", emsg)
# alright, manhole has started
self._manhole_service.started = datetime.utcnow()
self._manhole_service.status = 'started'
started_topic = '{}.on_manhole_started'.format(self._uri_prefix)
started_info = self._manhole_service.marshal()
self.publish(started_topic, started_info, options=PublishOptions(exclude=details.caller))
returnValue(started_info)
@inlineCallbacks
def stop_manhole(self, details=None):
"""
Stop the Manhole service running in this process.
This procedure is registered under
* ``crossbar.node.<node_id>.worker.<worker_id>.stop_manhole`` for native workers and under
* ``crossbar.node.<node_id>.controller.stop_manhole`` for node controllers
When no Manhole service is currently running within this process,
or the Manhole service is already shutting down, a
``crossbar.error.not_started`` WAMP error is raised.
The procedure will publish an event when the service **is stopping** to
* ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_stopping`` for native workers and
* ``crossbar.node.<node_id>.controller.on_manhole_stopping`` for node controllers
and will publish an event when the service **has stopped** to
* ``crossbar.node.<node_id>.worker.<worker_id>.on_manhole_stopped`` for native workers and
* ``crossbar.node.<node_id>.controller.on_manhole_stopped`` for node controllers
"""
self.log.debug("{cls}.stop_manhole", cls=self.__class__.__name__)
if not self._manhole_service or self._manhole_service.status != 'started':
emsg = "Cannot stop manhole: not running (or already shutting down)"
raise ApplicationError(u"crossbar.error.not_started", emsg)
self._manhole_service.status = 'stopping'
stopping_topic = u'{}.on_manhole_stopping'.format(self._uri_prefix)
stopping_info = None
# the caller gets a progressive result ..
if details.progress:
details.progress(stopping_info)
# .. while all others get an event
self.publish(stopping_topic, stopping_info, options=PublishOptions(exclude=details.caller))
try:
yield self._manhole_service.port.stopListening()
except Exception as e:
self.log.warn("error while stop listening on endpoint: {error}", error=e)
self._manhole_service = None
stopped_topic = u'{}.on_manhole_stopped'.format(self._uri_prefix)
stopped_info = None
self.publish(stopped_topic, stopped_info, options=PublishOptions(exclude=details.caller))
returnValue(stopped_info)
def get_manhole(self, details=None):
"""
Get current manhole service information.
:returns: dict -- A dict with service information or `None` if the service is not running.
"""
self.log.debug("{cls}.get_manhole", cls=self.__class__.__name__)
if not _HAS_MANHOLE:
emsg = "Could not start manhole: required packages are missing ({})".format(_MANHOLE_MISSING_REASON)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.feature_unavailable", emsg)
if not self._manhole_service:
return None
else:
return self._manhole_service.marshal()
def utcnow(self, details=None):
"""
Return current time as determined from within this process.
**Usage:**
This procedure is registered under
* ``crossbar.node.<node_id>.worker.<worker_id>.utcnow`` for native workers and under
* ``crossbar.node.<node_id>.controller.utcnow`` for node controllers
:returns: Current time (UTC) in UTC ISO 8601 format.
:rtype: unicode
"""
self.log.debug("{cls}.utcnow", cls=self.__class__.__name__)
return utcnow()
def started(self, details=None):
"""
Return start time of this process.
**Usage:**
This procedure is registered under
* ``crossbar.node.<node_id>.worker.<worker_id>.started`` for native workers and under
* ``crossbar.node.<node_id>.controller.started`` for node controllers
:returns: Start time (UTC) in UTC ISO 8601 format.
:rtype: unicode
"""
self.log.debug("{cls}.started", cls=self.__class__.__name__)
return utcstr(self._started)
def uptime(self, details=None):
"""
Return uptime of this process.
**Usage:**
This procedure is registered under
* ``crossbar.node.<node_id>.worker.<worker_id>.uptime`` for native workers and under
* ``crossbar.node.<node_id>.controller.uptime`` for node controllers
:returns: Uptime in seconds.
:rtype: float
"""
self.log.debug("{cls}.uptime", cls=self.__class__.__name__)
now = datetime.utcnow()
return (now - self._started).total_seconds()
class FileUploadResource(Resource):
"""
Twisted Web resource that handles file uploads over `HTTP/POST` requests.
"""
log = make_logger()
def __init__(self,
upload_directory,
temp_directory,
form_fields,
upload_session,
options=None):
"""
:param upload_directory: The target directory where uploaded files will be stored.
:type upload_directory: str
:param temp_directory: A temporary directory where chunks of a file being uploaded are stored.
:type temp_directory: str
:param form_fields: Names of HTML form fields used for uploading.
:type form_fields: dict
:param upload_session: An instance of `ApplicationSession` used for publishing progress events.
:type upload_session: obj
:param options: Options for file upload.
:type options: dict or None
"""
Resource.__init__(self)
self._uploadRoot = upload_directory
self._tempDirRoot = temp_directory
self._form_fields = form_fields
self._fileupload_session = upload_session
self._options = options or {}
self._max_file_size = self._options.get('max_file_size',
10 * 1024 * 1024)
self._fileTypes = self._options.get('file_types', None)
self._file_permissions = self._options.get('file_permissions', None)
# track uploaded files / chunks
self._uploads = {}
self.log.info('Upload Resource started.')
# scan the temp dir for uploaded chunks and fill the _uploads dict with it
# so existing uploads can be resumed
# all other remains will be purged
for _fileTempDir in os.listdir(self._tempDirRoot):
fileTempDir = os.path.join(self._tempDirRoot, _fileTempDir)
fileTempName = os.path.basename(fileTempDir)
if os.path.isdir(fileTempDir):
self._uploads[fileTempName] = {
'chunk_list': [],
'origin': 'startup'
}
for chunk in os.listdir(fileTempDir):
if chunk[:6] == 'chunk_':
self._uploads[fileTempName]['chunk_list'].append(
int(chunk[6:]))
else:
os.remove(os.path.join(fileTempDir, chunk))
# if no chunks detected then remove remains completely
if len(self._uploads[fileTempName]['chunk_list']) == 0:
shutil.rmtree(fileTempDir)
self._uploads.pop(fileTempName, None)
else: # fileTempDir is a file remaining from a single chunk upload
os.remove(fileTempDir)
self.log.debug("Scanned pending uploads: {uploads}",
uploads=self._uploads)
def render_POST(self, request):
headers = {
x.decode('iso-8859-1'): y.decode('iso-8859-1')
for x, y in request.getAllHeaders().items()
}
origin = headers['host']
postFields = cgi.FieldStorage(fp=request.content,
headers=headers,
environ={"REQUEST_METHOD": "POST"})
f = self._form_fields
filename = postFields[f['file_name']].value
totalSize = int(postFields[f['total_size']].value)
totalChunks = int(postFields[f['total_chunks']].value)
chunkSize = int(postFields[f['chunk_size']].value)
chunkNumber = int(postFields[f['chunk_number']].value)
fileContent = postFields[f['content']].value
if 'chunk_extra' in f and f['chunk_extra'] in postFields:
chunk_extra = json.loads(postFields[f['chunk_extra']].value)
else:
chunk_extra = {}
if 'finish_extra' in f and f['finish_extra'] in postFields:
finish_extra = json.loads(postFields[f['finish_extra']].value)
else:
finish_extra = {}
fileId = filename
# # prepare user specific upload areas
# # NOT YET IMPLEMENTED
# #
# if 'auth_id' in f and f['auth_id'] in postFields:
# auth_id = postFields[f['auth_id']].value
# mydir = os.path.join(self._uploadRoot, auth_id)
# my_temp_dir = os.path.join(self._tempDirRoot, auth_id)
#
# # check if auth_id is a valid directory_name
# #
# if auth_id != auth_id.encode('ascii', 'ignore'):
# msg = "The requestor auth_id must be an ascii string."
# if self._debug:
# log.msg(msg)
# # 415 Unsupported Media Type
# request.setResponseCode(415, msg)
# return msg
# else:
# auth_id = 'anonymous'
# create user specific folder
# mydir = self._uploadRoot
# my_temp_dir = self._tempDirRoot
# if not os.path.exists(mydir):
# os.makedirs(mydir)
# if not os.path.exists(my_temp_dir):
# os.makedirs(my_temp_dir)
# prepare the on_progress publisher
if 'on_progress' in f and f['on_progress'] in postFields and self._fileupload_session != {}:
topic = postFields[f['on_progress']].value
if 'session' in f and f['session'] in postFields:
session = int(postFields[f['session']].value)
publish_options = PublishOptions(eligible=[session])
else:
publish_options = None
def fileupload_publish(payload):
self._fileupload_session.publish(topic,
payload,
options=publish_options)
else:
def fileupload_publish(payload):
pass
# Register upload right at the start to avoid overlapping upload conflicts
#
if fileId not in self._uploads:
self._uploads[fileId] = {'chunk_list': [], 'origin': origin}
chunk_is_first = True
self.log.debug(
'Started upload of file: file_name={file_name}, total_size={total_size}, total_chunks={total_chunks}, chunk_size={chunk_size}, chunk_number={chunk_number}',
file_name=fileId,
total_size=totalSize,
total_chunks=totalChunks,
chunk_size=chunkSize,
chunk_number=chunkNumber)
else:
chunk_is_first = False
# If the chunks are read at startup of crossbar any client may claim and resume the pending upload !
#
upl = self._uploads[fileId]
if upl['origin'] == 'startup':
self.log.debug(
'Will try to resume upload of file: file_name={file_name}, total_size={total_size}, total_chunks={total_chunks}, chunk_size={chunk_size}, chunk_number={chunk_number}',
file_name=fileId,
total_size=totalSize,
total_chunks=totalChunks,
chunk_size=chunkSize,
chunk_number=chunkNumber)
upl['origin'] = origin
else:
# check if another session is uploading this file already
#
if upl['origin'] != origin:
msg = "File being uploaded is already uploaded in a different session."
self.log.debug(msg)
# 409 Conflict
request.setResponseCode(409, msg.encode('utf8'))
return msg.encode('utf8')
else:
# check if the chunk is being uploaded in this very session already
# this should never happen !
if chunkNumber in upl['chunk_list']:
msg = "Chunk beeing uploaded is already uploading."
self.log.debug(msg)
# Don't throw a conflict. This may be a wanted behaviour.
# Even if an upload would be resumable, you don't have to resume.
# 409 Conflict
# request.setResponseCode(409, msg.encode('utf8'))
# return msg.encode('utf8')
# check file size
#
if totalSize > self._max_file_size:
msg = "Size {} of file to be uploaded exceeds maximum {}".format(
totalSize, self._max_file_size)
self.log.debug(msg)
# 413 Request Entity Too Large
request.setResponseCode(413, msg.encode('utf8'))
return msg.encode('utf8')
# check file extensions
#
extension = os.path.splitext(filename)[1]
if self._fileTypes and extension not in self._fileTypes:
msg = "Type '{}' of file to be uploaded is in allowed types {}".format(
extension, self._fileTypes)
self.log.debug(msg)
# 415 Unsupported Media Type
request.setResponseCode(415, msg.encode('utf8'))
return msg.encode('utf8')
# TODO: check mime type
#
fileTempDir = os.path.join(self._tempDirRoot, fileId)
chunkName = os.path.join(fileTempDir, 'chunk_' + str(chunkNumber))
_chunkName = os.path.join(
fileTempDir,
'#kfhf3kz412uru578e38viokbjhfvz4w__' + 'chunk_' + str(chunkNumber))
def mergeFile():
# every chunk has to check if it is the last chunk written, except in a single chunk scenario
if totalChunks > 1 and len(
self._uploads[fileId]['chunk_list']) >= totalChunks:
# last chunk
self.log.debug(
'Finished file upload after chunk {chunk_number} with chunk_list {chunk_list}',
chunk_number=chunkNumber,
chunk_list=self._uploads)
# Merge all files into one file and remove the temp files
# TODO: How to avoid the extra file IO ?
finalFileName = os.path.join(self._uploadRoot, fileId)
_finalFileName = os.path.join(
fileTempDir, '#kfhf3kz412uru578e38viokbjhfvz4w__' + fileId)
with open(_finalFileName, 'wb') as _finalFile:
for cn in range(1, totalChunks + 1):
with open(
os.path.join(fileTempDir, 'chunk_' + str(cn)),
'rb') as ff:
_finalFile.write(ff.read())
os.rename(_finalFileName, finalFileName)
if self._file_permissions:
perm = int(self._file_permissions, 8)
try:
os.chmod(finalFileName, perm)
except Exception as e:
msg = "file upload resource - could not change file permissions of uploaded file"
self.log.debug(msg)
self.log.debug(e)
self._uploads.pop(fileId, None)
request.setResponseCode(500, msg.encode('utf8'))
return msg.encode('utf8')
else:
self.log.debug(
"Changed permissions on {file_name} to {permissions}",
file_name=finalFileName,
permissions=self._file_permissions)
# remove the file temp folder
shutil.rmtree(fileTempDir)
self._uploads.pop(fileId, None)
# publish file upload progress to file_progress_URI
fileupload_publish({
"id": fileId,
"chunk": chunkNumber,
"name": filename,
"total": totalSize,
"remaining": 0,
"status": "finished",
"progress": 1.,
"finish_extra": finish_extra,
"chunk_extra": chunk_extra
})
if chunk_is_first:
# first chunk of file
# publish file upload start
#
fileupload_publish({
"id": fileId,
"chunk": chunkNumber,
"name": filename,
"total": totalSize,
"remaining": totalSize,
"status": "started",
"progress": 0.,
"chunk_extra": chunk_extra
})
if totalChunks == 1:
# only one chunk overall -> write file directly
finalFileName = os.path.join(self._uploadRoot, fileId)
_finalFileName = os.path.join(
self._tempDirRoot,
'#kfhf3kz412uru578e38viokbjhfvz4w__' + fileId)
with open(_finalFileName, 'wb') as _finalFile:
_finalFile.write(fileContent)
if self._file_permissions:
perm = int(self._file_permissions, 8)
try:
os.chmod(_finalFileName, perm)
except Exception as e:
# finalFileName.remove()
msg = "Could not change file permissions of uploaded file"
self.log.debug(msg)
self.log.debug(e)
request.setResponseCode(500, msg.encode('utf8'))
return msg.encode('utf8')
else:
self.log.debug(
"Changed permissions on {file_name} to {permissions}",
file_name=finalFileName,
permissions=self._file_permissions)
os.rename(_finalFileName, finalFileName)
if chunkNumber not in self._uploads[fileId]['chunk_list']:
self._uploads[fileId]['chunk_list'].append(chunkNumber)
self._uploads.pop(fileId, None)
# publish file upload progress to file_progress_URI
fileupload_publish({
"id": fileId,
"chunk": chunkNumber,
"name": filename,
"total": totalSize,
"remaining": 0,
"status": "finished",
"progress": 1.,
"finish_extra": finish_extra,
"chunk_extra": chunk_extra
})
else:
# first of more chunks
# fileTempDir.remove() # any potential conflict should have been resolved above. This should not be necessary!
if not os.path.isdir(fileTempDir):
os.makedirs(fileTempDir)
with open(_chunkName, 'wb') as chunk:
chunk.write(fileContent)
os.rename(_chunkName,
chunkName) # atomic file system operation
self.log.debug('chunk_' + str(chunkNumber) +
' written and moved to ' + chunkName)
# publish file upload progress
#
fileupload_publish({
"id":
fileId,
"chunk":
chunkNumber,
"name":
filename,
"total":
totalSize,
"remaining":
totalSize - chunkSize,
"status":
"progress",
"progress":
round(float(chunkSize) / float(totalSize), 3),
"chunk_extra":
chunk_extra
})
if chunkNumber not in self._uploads[fileId]['chunk_list']:
self._uploads[fileId]['chunk_list'].append(chunkNumber)
mergeFile()
# clean the temp dir once per file upload
self._remove_stale_uploads()
else:
# intermediate chunk
if not os.path.isdir(fileTempDir):
os.makedirs(fileTempDir)
with open(_chunkName, 'wb') as chunk:
chunk.write(fileContent)
os.rename(_chunkName, chunkName)
self.log.debug('chunk_' + str(chunkNumber) +
' written and moved to ' + chunkName)
if chunkNumber not in self._uploads[fileId]['chunk_list']:
self._uploads[fileId]['chunk_list'].append(chunkNumber)
received = sum(
os.path.getsize(os.path.join(fileTempDir, f))
for f in os.listdir(fileTempDir))
fileupload_publish({
"id":
fileId,
"chunk":
chunkNumber,
"name":
filename,
"total":
totalSize,
"remaining":
totalSize - received,
"status":
"progress",
"progress":
round(float(received) / float(totalSize), 3),
"chunk_extra":
chunk_extra
})
mergeFile()
# no errors encountered -> respond success
request.setResponseCode(200)
return b''
def _remove_stale_uploads(self):
"""
This only works if there is a temp folder exclusive for crossbar file uploads
if the system temp folder is used then crossbar creates a "crossbar-uploads" there and
uses that as the temp folder for uploads
If you don't clean up regularly an attacker could fill up the OS file system
"""
for _fileTempDir in os.listdir(self._tempDirRoot):
fileTempDir = os.path.join(self._tempDirRoot, _fileTempDir)
self.log.debug('REMOVE STALE UPLOADS ' +
str(os.path.basename(fileTempDir)))
if os.path.isdir(fileTempDir) and os.path.basename(
fileTempDir) not in self._uploads:
shutil.rmtree(fileTempDir)
def render_GET(self, request):
"""
This method can be used to check whether a chunk has been uploaded already.
It returns with HTTP status code `200` if yes and `404` if not.
The request needs to contain the file identifier and the chunk number to check for.
"""
for param in ['file_name', 'chunk_number']:
if not self._form_fields[param].encode(
'iso-8859-1') in request.args:
msg = "file upload resource - missing request query parameter '{}', configured from '{}'".format(
self._form_fields[param], param)
self.log.debug(msg)
# 400 Bad Request
request.setResponseCode(400, msg.encode('utf8'))
return msg.encode('utf8')
file_name = request.args[self._form_fields['file_name'].encode(
'iso-8859-1')][0].decode('utf8')
chunk_number = int(
request.args[self._form_fields['chunk_number'].encode(
'iso-8859-1')][0].decode('utf8'))
# a complete upload will be repeated an incomplete upload will be resumed
if file_name in self._uploads and chunk_number in self._uploads[
file_name]['chunk_list']:
self.log.debug(
"Skipping chunk upload {file_name} of chunk {chunk_number}",
file_name=file_name,
chunk_number=chunk_number)
msg = b"chunk of file already uploaded"
request.setResponseCode(200, msg)
return msg
else:
msg = b"chunk of file not yet uploaded"
request.setResponseCode(404, msg)
return msg