def run():
def on_progress(data):
print(f'on_progress: {len(data)}')
global progress_file
progress_file += data['data']
if progressive:
print(f'Progressive call to method {uri} with {_args}, {_kwargs}')
options = CallOptions(on_progress=on_progress)
else:
print(f"call method {uri} with {_args}, {_kwargs}")
options = CallOptions()
try:
ret = yield session.call(uri, *_args, **_kwargs, options=options)
except Exception as e:
print(e)
else:
try:
ret = json.dumps(ret, indent=4)
except Exception as e:
pass
else:
print(ret)
if tofile:
with open(tofile, 'w') as f:
f.write(ret)
global progress_file
if progress_file:
with open(tofile, 'wb') as f:
data = base64.b64decode(progress_file)
f.write(data)
session.leave()
def _configure_native_worker_common(self, worker_logname, worker_id,
worker):
# expanding PYTHONPATH of the newly started worker is now done
# directly in NodeController._start_native_worker
worker_options = worker.get('options', {})
if False:
if 'pythonpath' in worker_options:
added_paths = yield self._controller.call(
u'crossbar.worker.{}.add_pythonpath'.format(worker_id),
worker_options['pythonpath'],
options=CallOptions())
self.log.warn("{worker}: PYTHONPATH extended for {paths}",
worker=worker_logname,
paths=added_paths)
# FIXME: as the CPU affinity is in the worker options, this _also_ (see above fix)
# should be done directly in NodeController._start_native_worker
if True:
if 'cpu_affinity' in worker_options:
new_affinity = yield self._controller.call(
u'crossbar.worker.{}.set_cpu_affinity'.format(worker_id),
worker_options['cpu_affinity'],
options=CallOptions())
self.log.debug("{worker}: CPU affinity set to {affinity}",
worker=worker_logname,
affinity=new_affinity)
# this is fine to start after the worker has been started, as manhole is
# CB developer/support feature anyways (like a vendor diagnostics port)
if 'manhole' in worker:
yield self._controller.call(
u'crossbar.worker.{}.start_manhole'.format(worker_id),
worker['manhole'],
options=CallOptions())
self.log.debug("{worker}: manhole started", worker=worker_logname)
def __init__(self, client):
self.client = client
self.NeuroArchWrite_rpc = partial(client.rpc,
'ffbo.na.NeuroArch.write.{}'.format(client.naServerID),
options=CallOptions(timeout=10000))
self.NeuroArchQuery_rpc = partial(client.rpc,
'ffbo.na.NeuroArch.query.{}'.format(client.naServerID),
options=CallOptions(timeout=10000))
def _configure_native_worker_proxy(self, worker_logname, worker_id, worker):
yield self._configure_native_worker_common(worker_logname, worker_id, worker)
# start transports on proxy
for i, transport in enumerate(worker.get('transports', [])):
if 'id' in transport:
transport_id = transport['id']
else:
transport_id = 'transport{:03d}'.format(i)
transport['id'] = transport_id
self.log.info(
"Order {worker_logname} to start Transport {transport_id}",
worker_logname=worker_logname,
transport_id=hlid(transport_id),
)
# XXX we're doing startup, and begining proxy workers --
# want to share the web-transport etc etc stuff between
# these and otehr kinds of routers / transports
yield self._controller.call(u'crossbar.worker.{}.start_proxy_transport'.format(worker_id),
transport_id,
transport,
options=CallOptions())
self.log.info(
"Ok, {worker_logname} has started Transport {transport_id}",
worker_logname=worker_logname,
transport_id=hlid(transport_id),
)
for i, backend in enumerate(worker.get('backends', [])):
if 'id' in backend:
backend_id = backend['id']
else:
backend_id = 'backend{:03d}'.format(i)
backend['id'] = backend_id
self.log.info(
"Order {worker_logname} to start BackendTransport {backend_id}",
worker_logname=worker_logname,
backend_id=hlid(backend_id),
)
yield self._controller.call(u'crossbar.worker.{}.start_proxy_backend'.format(worker_id),
backend_id,
backend,
options=CallOptions())
self.log.info(
"Ok, {worker_logname} has started BackendTransport {backend_id}",
worker_logname=worker_logname,
transport_id=hlid(backend_id),
)
def onJoin(self, details):
print("session joined")
# ---- Publisher ----
for i in range(5):
msg = u'Testing %d' % i
print("Publishing:", msg)
self.publish(topic, msg)
yield sleep(1)
# ---- Caller ----
print('Calling procedure:', procedure)
try:
res = yield self.call(procedure, 2, 3)
print("call result: {}".format(res))
except Exception as e:
print("call error: {0}".format(e))
def on_prog(msg):
print("here")
print("partial result: {}".format(msg))
print('Calling procedure:', procedure2)
try:
res = yield self.call(procedure2, 'A',
options=CallOptions(on_progress=on_prog))
print("call result: {}".format(res))
except Exception as e:
print("call error: {0}".format(e))
def on_call(*args, **kwargs):
assert 'details' in kwargs
details = kwargs.pop('details')
options = kwargs.pop('options', None)
if details.caller is None or details.caller_authrole is None or details.caller_authid is None:
raise RuntimeError(
"Internal error attempting rlink forwarding"
)
self.log.info(
'Received invocation on uri={uri}, options={options} (caller={caller}, caller_authid={caller_authid}, caller_authrole={caller_authrole}, forward_for={forward_for})',
uri=uri,
options=options,
caller=details.caller,
caller_authid=details.caller_authid,
caller_authrole=details.caller_authrole,
forward_for=details.forward_for)
this_forward = {
'session': details.caller,
'authid': details.caller_authrole,
'authrole': details.caller_authrole,
}
if details.forward_for:
# the call comes already forwarded from a router node ..
if len(details.forward_for) >= 0:
self.log.debug('SKIP! already forwarded')
return
forward_for = copy.deepcopy(details.forward_for)
forward_for.append(this_forward)
else:
forward_for = [this_forward]
options = CallOptions(forward_for=forward_for)
try:
result = yield self.call(uri, *args, options=options, **kwargs)
except TransportLost:
return
except ApplicationError as e:
if e.error not in ['wamp.close.normal']:
self.log.warn('FAILED TO CALL 1: {} {}'.format(type(e), str(e)))
return
except Exception as e:
if not ERR_MSG[0]:
self.log.warn('FAILED TO CALL 2: {} {}'.format(type(e), str(e)))
ERR_MSG[0] = True
return
self.log.info(
"RLink forward-invoked call {dir} (options={options})",
dir=self.DIR,
options=options,
)
return result
async def call(self,
endpoint: str,
*args,
on_progress: Callable[..., None] = None,
timeout: int = None,
**kwargs):
"""
Call an RPC function.
:param endpoint: name of the endpoint to call
:param args: positional arguments to call the endpoint with
:param on_progress: a callable that will receive progress reports from the endpoint
:param timeout: timeout (in seconds) to wait for the completion of the call
:param kwargs: keyword arguments to call the endpoint with
:return: the return value of the call
:raises TimeoutError: if the call times out
"""
assert check_argument_types()
if self._session is None:
await self.connect()
options = CallOptions(on_progress=on_progress, timeout=timeout)
return await self._session.call(endpoint,
*args,
options=options,
**kwargs)
def __init__(self, config=None):
ApplicationSession.__init__(self, config)
global live
live = self
self.logger = logging.getLogger('Live')
self.logger.info("Config: %s", config)
self.account_id = config.extra['authid']
self.secret = config.extra['secret']
if '-' not in self.account_id:
self.account_id = "local-%s" % self.account_id
self.authid = '%s:%s' % (self.account_id, self.secret[-7:])
self.joined = False
self.lock = DeferredLock()
self.checks = {}
self.workers = {}
self.CallOptions = CallOptions()
def call_progress(topic):
print("calling '{}' progressively".format(topic))
def on_progress(some_data):
print("received: '{}'".format(some_data))
return session.call(topic,
options=CallOptions(on_progress=on_progress))
def select_DataSource(self, name, version = None):
uri = "ffbo.na.datasource.{}".format(self.client.naServerID)
res = self.client.rpc(
uri,
name, version = version,
options=CallOptions(timeout=10000) )
result = self._check_result(res)
return result
def executeNAquery(self,
res,
language='en',
uri=None,
queryID=None,
progressive=True,
threshold=5):
"""Execute an NA query.
Args:
res (dict): Neuroarch query.
language (str): Language to use.
uri (str): A custom FFBO query URI if desired.
queryID (str): Query ID to be used. Generated automatically.
progressive (bool): Whether the loading should be progressive. Needs to be true most of the time for connection to be stable.
threshold (int): Data chunk size. Low threshold is required for the connection to be stable.
Returns:
bool: Whether the process has been successful.
"""
def on_progress(x, res):
res.append(x)
if isinstance(res, str):
res = json.loads(res)
if uri == None:
uri = 'ffbo.na.query.' + self.naServerID
if "uri" in res.keys():
uri = res["uri"] + "." + self.naServerID
if queryID == None:
queryID = guidGenerator()
del self.data # Reset the data in the backend
self.data = []
res['queryID'] = queryID
res['threshold'] = threshold
res['data_callback_uri'] = 'ffbo.ui.receive_data'
res_list = []
res = self.client.session.call(
uri,
res,
options=CallOptions(
on_progress=partial(on_progress, res=res_list)))
a = {}
a['data'] = res
a['messageType'] = 'Data'
a['widget'] = 'NLP'
if "retrieve_tag" in uri:
a['messageType'] = 'TagData'
self.tryComms(a)
self.executeNAquery({"command": {"retrieve": {"state": 0}}})
if progressive == True:
self.tryComms(a)
self.data.append(a)
return self.data
else:
self.tryComms(a)
return a
def onJoin(self, details):
def on_progress(i):
print("Progress: {}".format(i))
res = yield from self.call('com.myapp.longop', 3, options=CallOptions(onProgress=on_progress))
print("Final: {}".format(res))
self.leave()
def onJoin(self, details):
print("session attached")
def on_progress(i):
print("Progress: {}".format(i))
res = yield self.call(u'com.myapp.longop', 3, options=CallOptions(on_progress=on_progress))
print("Final: {}".format(res))
self.leave()
def onJoin(self, details):
def on_event(val):
print("Someone requested to square non-positive: {}".format(val))
yield from self.subscribe(on_event, 'com.myapp.square_on_nonpositive')
for val in [2, 0, -2]:
res = yield from self.call('com.myapp.square', val, options=CallOptions(disclose_me=True))
print("Squared {} = {}".format(val, res))
yield from self.leave()
async def onJoin(self, details):
def on_event(val):
print("Someone requested to square non-positive: {}".format(val))
await self.subscribe(on_event, u'com.myapp.square_on_nonpositive')
for val in [2, 0, -2]:
res = await self.call(u'com.myapp.square', val, options=CallOptions())
print("Squared {} = {}".format(val, res))
await self.leave()
def _test_rpc_errors(self):
options = CallOptions()
for proc in [(u'com.myapp.failme', False), (u'com.myapp.failme', True),
(u'com.myapp.encrypted.failme', False),
(u'com.myapp.encrypted.failme', True)]:
try:
res = yield self.call(proc[0], proc[1], options=options)
print("{} call with {} - result: {}".format(
proc[0], proc[1], res))
except Exception as e:
print("{} call with {} - error: {}".format(
proc[0], proc[1], e))
def onJoin(self, details):
print("session attached")
def on_event(val):
print("Someone requested to square non-positive: {}".format(val))
yield self.subscribe(on_event, 'com.myapp.square_on_nonpositive')
for val in [2, 0, -2]:
res = yield self.call('com.myapp.square',
val,
options=CallOptions())
print("Squared {} = {}".format(val, res))
self.leave()
def _test_rpc(self, delay=None):
options = CallOptions()
counter = 1
while counter <= self.NUM:
for proc in [u'com.myapp.add2', u'com.myapp.encrypted.add2']:
try:
res = yield self.call(proc, 23, counter, options=options)
print("{} call result: {}".format(proc, res))
except Exception as e:
print("{} call error: {}".format(proc, e))
if delay:
yield sleep(delay)
counter += 1
def _configure_native_worker_container(self, worker_logname, worker_id,
worker):
yield self._configure_native_worker_common(worker_logname, worker_id,
worker)
# if components exit "very soon after" we try to start them,
# we consider that a failure and shut our node down. We remove
# this subscription 2 seconds after we're done starting
# everything (see below). This is necessary as start_component
# returns as soon as we've established a connection to the
# component
def component_exited(info):
component_id = info.get("id")
self.log.critical(
"Component '{component_id}' failed to start; shutting down node.",
component_id=component_id)
try:
self._reactor.stop()
except twisted.internet.error.ReactorNotRunning:
pass
topic = u'crossbar.worker.{}.container.on_component_stop'.format(
worker_id)
component_stop_sub = yield self._controller.subscribe(
component_exited, topic)
# start components to run embedded in the container
#
for component in worker.get('components', []):
if 'id' in component:
component_id = component['id']
else:
component_id = 'component{:03d}'.format(self._component_no)
component['id'] = component_id
self._component_no += 1
yield self._controller.call(
u'crossbar.worker.{}.start_component'.format(worker_id),
component_id,
component,
options=CallOptions())
self.log.info("{worker}: component '{component_id}' started",
worker=worker_logname,
component_id=component_id)
# after 2 seconds, consider all the application components running
self._reactor.callLater(2, component_stop_sub.unsubscribe)
def neuroarch_query(request, details=None):
"""
Call the neuroarch_query module with a neuroarch-json object
The request should have
user: session_id for client
server: session_id for na server
query: neuroarch json query object
"""
try:
request['user'] = details.caller
user_details = yield self.call(
six.u('ffbo.auth_server.get_user'), details.caller)
if user_details: request['username'] = user_details['username']
self.log.info(
"neuroarch_query() accessed with request: {request}",
request=request)
progressive_result = {}
def on_progress(p):
progressive_result.update(p)
#print request
result = yield self.call(
six.u('ffbo.na.query.' + str(request['server'])),
request,
options=CallOptions(on_progress=on_progress))
self.log.info("na_query returned with result")
if progressive_result:
result = progressive_result
# Catch no results and return
if result == "{}":
raise ValueError('Neuroarch returned zero results')
returnValue(feedback_success(request, (result)))
except ValueError as e:
feedback = feedback_error(request,
"Neuroarch returned zero results", e)
returnValue(feedback)
except ApplicationError as e:
raise e
feedback = feedback_error(server_id,
"Unable to contact server", e)
returnValue(feedback)
def _test_rpc(self, delay=None):
options = CallOptions()
counter = 1
while counter <= self.NUM:
for proc in ['com.myapp.add2', 'com.myapp.encrypted.add2']:
try:
res = yield self.call(proc, 23, counter, options=options)
self.log.info('{proc} call result: {res}',
proc=proc,
res=res)
except Exception as e:
self.log.info('{proc} call error: {e}', proc=proc, e=e)
if delay:
yield sleep(delay)
counter += 1
async def worker(index, cid):
class Data():
def __init__(self):
self.data = bytes()
def progress(self, update):
self.data += bytes(update)
#get the binary data
uri = f"ocp.documents.{self.onlineDoc.id}.raw.BinaryByCid"
dat = Data()
opt = CallOptions(on_progress=dat.progress)
result = await self.onlineDoc.connection.api.call(
uri, cid, options=opt)
if result is not None:
dat.progress(result)
values[index] = dat.data
def _configure_native_worker_websocket_testee(self, worker_logname, worker_id, worker):
yield self._configure_native_worker_common(worker_logname, worker_id, worker)
# start transport on websocket-testee
transport = worker['transport']
transport_id = 'transport{:03d}'.format(self._transport_no)
transport['id'] = transport_id
self._transport_no = 1
yield self._controller.call('crossbar.worker.{}.start_websocket_testee_transport'.format(worker_id),
transport_id,
transport,
options=CallOptions())
self.log.info(
"{logname}: transport '{tid}' started",
logname=worker_logname,
tid=transport_id,
)
def _test_rpc_errors(self):
options = CallOptions()
for proc, encrypted_error in [('com.myapp.failme', False),
('com.myapp.failme', True),
('com.myapp.encrypted.failme', False),
('com.myapp.encrypted.failme', True)]:
try:
res = yield self.call(proc, encrypted_error, options=options)
self.log.info(
'{proc} called with encrypted_error={encrypted_error} - result: {res}',
proc=proc,
encrypted_error=encrypted_error,
res=res)
except Exception as e:
self.log.info(
'{proc} called with encrypted_error={encrypted_error} - error: {e}',
proc=proc,
encrypted_error=encrypted_error,
e=e)
def _configure_native_worker_container(self, worker_logname, worker_id, worker):
yield self._configure_native_worker_common(worker_logname, worker_id, worker)
# start components to run embedded in the container
#
for component in worker.get('components', []):
if 'id' in component:
component_id = component['id']
else:
component_id = 'component{:03d}'.format(self._component_no)
component['id'] = component_id
self._component_no += 1
yield self._controller.call('crossbar.worker.{}.start_component'.format(worker_id),
component_id,
component,
options=CallOptions())
self.log.info("{worker}: component '{component_id}' started",
worker=worker_logname,
component_id=component_id)
def _startup(self, config):
"""
Startup elements in the node as specified in the provided node configuration.
"""
self.log.info('Configuring node from local configuration ...')
# call options we use to call into the local node management API
call_options = CallOptions()
# fake call details we use to call into the local node management API
call_details = CallDetails(caller=0)
# get contoller configuration subpart
controller = config.get('controller', {})
# start Manhole in node controller
if 'manhole' in controller:
yield self._controller.start_manhole(controller['manhole'],
details=call_details)
# startup all workers
workers = config.get('workers', [])
if len(workers):
self.log.info('Starting {nworkers} workers ...',
nworkers=len(workers))
else:
self.log.info('No workers configured!')
for worker in workers:
# worker ID
if 'id' in worker:
worker_id = worker.pop('id')
else:
worker_id = 'worker-{:03d}'.format(self._worker_no)
self._worker_no += 1
# worker type - a type of working process from the following fixed list
worker_type = worker['type']
assert (worker_type
in ['router', 'container', 'guest', 'websocket-testee'])
# set logname depending on worker type
if worker_type == 'router':
worker_logname = "Router '{}'".format(worker_id)
elif worker_type == 'container':
worker_logname = "Container '{}'".format(worker_id)
elif worker_type == 'websocket-testee':
worker_logname = "WebSocketTestee '{}'".format(worker_id)
elif worker_type == 'guest':
worker_logname = "Guest '{}'".format(worker_id)
else:
raise Exception("logic error")
# any worker specific options
worker_options = worker.get('options', {})
# native worker processes: router, container, websocket-testee
if worker_type in ['router', 'container', 'websocket-testee']:
# start a new native worker process ..
if worker_type == 'router':
yield self._controller.start_router(worker_id,
worker_options,
details=call_details)
elif worker_type == 'container':
yield self._controller.start_container(
worker_id, worker_options, details=call_details)
elif worker_type == 'websocket-testee':
yield self._controller.start_websocket_testee(
worker_id, worker_options, details=call_details)
else:
raise Exception("logic error")
# setup native worker generic stuff
if 'pythonpath' in worker_options:
added_paths = yield self._controller.call(
'crossbar.worker.{}.add_pythonpath'.format(worker_id),
worker_options['pythonpath'],
options=call_options)
self.log.debug("{worker}: PYTHONPATH extended for {paths}",
worker=worker_logname,
paths=added_paths)
if 'cpu_affinity' in worker_options:
new_affinity = yield self._controller.call(
'crossbar.worker.{}.set_cpu_affinity'.format(
worker_id),
worker_options['cpu_affinity'],
options=call_options)
self.log.debug("{worker}: CPU affinity set to {affinity}",
worker=worker_logname,
affinity=new_affinity)
if 'manhole' in worker:
yield self._controller.call(
'crossbar.worker.{}.start_manhole'.format(worker_id),
worker['manhole'],
options=call_options)
self.log.debug("{worker}: manhole started",
worker=worker_logname)
# setup router worker
if worker_type == 'router':
# start realms on router
for realm in worker.get('realms', []):
# start realm
if 'id' in realm:
realm_id = realm.pop('id')
else:
realm_id = 'realm-{:03d}'.format(self._realm_no)
self._realm_no += 1
yield self._controller.call(
'crossbar.worker.{}.start_router_realm'.format(
worker_id),
realm_id,
realm,
options=call_options)
self.log.info(
"{worker}: realm '{realm_id}' (named '{realm_name}') started",
worker=worker_logname,
realm_id=realm_id,
realm_name=realm['name'])
# add roles to realm
for role in realm.get('roles', []):
if 'id' in role:
role_id = role.pop('id')
else:
role_id = 'role-{:03d}'.format(self._role_no)
self._role_no += 1
yield self._controller.call(
'crossbar.worker.{}.start_router_realm_role'.
format(worker_id),
realm_id,
role_id,
role,
options=call_options)
self.log.info(
"{logname}: role '{role}' (named '{role_name}') started on realm '{realm}'",
logname=worker_logname,
role=role_id,
role_name=role['name'],
realm=realm_id,
)
# start uplinks for realm
for uplink in realm.get('uplinks', []):
if 'id' in uplink:
uplink_id = uplink.pop('id')
else:
uplink_id = 'uplink-{:03d}'.format(
self._uplink_no)
self._uplink_no += 1
yield self._controller.call(
'crossbar.worker.{}.start_router_realm_uplink'.
format(worker_id),
realm_id,
uplink_id,
uplink,
options=call_options)
self.log.info(
"{logname}: uplink '{uplink}' started on realm '{realm}'",
logname=worker_logname,
uplink=uplink_id,
realm=realm_id,
)
# start connections (such as PostgreSQL database connection pools)
# to run embedded in the router
for connection in worker.get('connections', []):
if 'id' in connection:
connection_id = connection.pop('id')
else:
connection_id = 'connection-{:03d}'.format(
self._connection_no)
self._connection_no += 1
yield self._controller.call(
'crossbar.worker.{}.start_connection'.format(
worker_id),
connection_id,
connection,
options=call_options)
self.log.info(
"{logname}: connection '{connection}' started",
logname=worker_logname,
connection=connection_id,
)
# start components to run embedded in the router
for component in worker.get('components', []):
if 'id' in component:
component_id = component.pop('id')
else:
component_id = 'component-{:03d}'.format(
self._component_no)
self._component_no += 1
yield self._controller.call(
'crossbar.worker.{}.start_router_component'.format(
worker_id),
component_id,
component,
options=call_options)
self.log.info(
"{logname}: component '{component}' started",
logname=worker_logname,
component=component_id,
)
# start transports on router
for transport in worker['transports']:
if 'id' in transport:
transport_id = transport.pop('id')
else:
transport_id = 'transport-{:03d}'.format(
self._transport_no)
self._transport_no += 1
yield self._controller.call(
'crossbar.worker.{}.start_router_transport'.format(
worker_id),
transport_id,
transport,
options=call_options)
self.log.info(
"{logname}: transport '{tid}' started",
logname=worker_logname,
tid=transport_id,
)
# setup container worker
elif worker_type == 'container':
# if components exit "very soon after" we try to
# start them, we consider that a failure and shut
# our node down. We remove this subscription 2
# seconds after we're done starting everything
# (see below). This is necessary as
# start_container_component returns as soon as
# we've established a connection to the component
def component_exited(info):
component_id = info.get("id")
self.log.critical(
"Component '{component_id}' failed to start; shutting down node.",
component_id=component_id)
try:
self._reactor.stop()
except twisted.internet.error.ReactorNotRunning:
pass
topic = 'crossbar.worker.{}.container.on_component_stop'.format(
worker_id)
component_stop_sub = yield self._controller.subscribe(
component_exited, topic)
# start connections (such as PostgreSQL database connection pools)
# to run embedded in the container
#
for connection in worker.get('connections', []):
if 'id' in connection:
connection_id = connection.pop('id')
else:
connection_id = 'connection-{:03d}'.format(
self._connection_no)
self._connection_no += 1
yield self._controller.call(
'crossbar.worker.{}.start_connection'.format(
worker_id),
connection_id,
connection,
options=call_options)
self.log.info(
"{logname}: connection '{connection}' started",
logname=worker_logname,
connection=connection_id,
)
# start components to run embedded in the container
#
for component in worker.get('components', []):
if 'id' in component:
component_id = component.pop('id')
else:
component_id = 'component-{:03d}'.format(
self._component_no)
self._component_no += 1
yield self._controller.call(
'crossbar.worker.{}.start_container_component'.
format(worker_id),
component_id,
component,
options=call_options)
self.log.info(
"{worker}: component '{component_id}' started",
worker=worker_logname,
component_id=component_id)
# after 2 seconds, consider all the application components running
self._reactor.callLater(2, component_stop_sub.unsubscribe)
# setup websocket-testee worker
elif worker_type == 'websocket-testee':
# start transport on websocket-testee
transport = worker['transport']
transport_id = 'transport-{:03d}'.format(
self._transport_no)
self._transport_no = 1
yield self._controller.call(
'crossbar.worker.{}.start_websocket_testee_transport'.
format(worker_id),
transport_id,
transport,
options=call_options)
self.log.info(
"{logname}: transport '{tid}' started",
logname=worker_logname,
tid=transport_id,
)
else:
raise Exception("logic error")
elif worker_type == 'guest':
# start guest worker
#
yield self._controller.start_guest(worker_id,
worker,
details=call_details)
self.log.info("{worker}: started", worker=worker_logname)
else:
raise Exception("logic error")
self.log.info('Local node configuration applied.')
def _configure_native_worker_router(self, worker_logname, worker_id,
worker):
yield self._configure_native_worker_common(worker_logname, worker_id,
worker)
# start realms on router
for realm in worker.get('realms', []):
# start realm
if 'id' in realm:
realm_id = realm['id']
else:
realm_id = 'realm{:03d}'.format(self._realm_no)
realm['id'] = realm_id
self._realm_no += 1
self.log.info(
"Order {worker_logname} to start Realm {realm_id}",
worker_logname=worker_logname,
realm_id=hlid(realm_id),
)
yield self._controller.call(
u'crossbar.worker.{}.start_router_realm'.format(worker_id),
realm_id,
realm,
options=CallOptions())
self.log.info(
"Ok, {worker_logname} has started Realm {realm_id}",
worker_logname=worker_logname,
realm_id=hlid(realm_id),
)
# add roles to realm
for role in realm.get('roles', []):
if 'id' in role:
role_id = role['id']
else:
role_id = 'role{:03d}'.format(self._role_no)
role['id'] = role_id
self._role_no += 1
self.log.info(
"Order Realm {realm_id} to start Role {role_id}",
realm_id=hlid(realm_id),
role_id=hlid(role_id),
)
yield self._controller.call(
u'crossbar.worker.{}.start_router_realm_role'.format(
worker_id),
realm_id,
role_id,
role,
options=CallOptions())
self.log.info(
"Ok, Realm {realm_id} has started Role {role_id}",
realm_id=hlid(realm_id),
role_id=hlid(role_id),
)
# start components to run embedded in the router
for component in worker.get('components', []):
if 'id' in component:
component_id = component['id']
else:
component_id = 'component{:03d}'.format(self._component_no)
component['id'] = component_id
self._component_no += 1
yield self._controller.call(
u'crossbar.worker.{}.start_router_component'.format(worker_id),
component_id,
component,
options=CallOptions())
self.log.info(
"{logname}: component '{component}' started",
logname=worker_logname,
component=component_id,
)
# start transports on router
for transport in worker.get('transports', []):
if 'id' in transport:
transport_id = transport['id']
else:
transport_id = 'transport{:03d}'.format(self._transport_no)
transport['id'] = transport_id
self._transport_no += 1
add_paths_on_transport_create = False
self.log.info(
"Order {worker_logname} to start Transport {transport_id}",
worker_logname=worker_logname,
transport_id=hlid(transport_id),
)
yield self._controller.call(
u'crossbar.worker.{}.start_router_transport'.format(worker_id),
transport_id,
transport,
create_paths=add_paths_on_transport_create,
options=CallOptions())
self.log.info(
"Ok, {worker_logname} has started Transport {transport_id}",
worker_logname=worker_logname,
transport_id=hlid(transport_id),
)
if not add_paths_on_transport_create:
if transport['type'] == 'web':
paths = transport.get('paths', {})
elif transport['type'] == 'universal':
paths = transport.get('web', {}).get('paths', {})
else:
paths = None
# Web service paths
if paths:
for path in sorted(paths):
if path != '/':
webservice = paths[path]
if 'id' in webservice:
webservice_id = webservice['id']
else:
webservice_id = 'webservice{:03d}'.format(
self._webservice_no)
webservice['id'] = webservice_id
self._webservice_no += 1
self.log.info(
"Order Transport {transport_id} to start Web Service {webservice_id}",
transport_id=hlid(transport_id),
webservice_id=hlid(webservice_id),
path=hluserid(path),
)
yield self._controller.call(
u'crossbar.worker.{}.start_web_transport_service'
.format(worker_id),
transport_id,
path,
webservice,
options=CallOptions())
self.log.info(
"Ok, Transport {transport_id} has started Web Service {webservice_id}",
transport_id=hlid(transport_id),
webservice_id=hlid(webservice_id),
path=hluserid(path),
)
def boot_from_config(self, config):
"""
Startup elements in the node as specified in the provided node configuration.
"""
# get controller configuration subpart
controller = config.get('controller', {})
parallel_worker_start = controller.get('options', {}).get(
'enable_parallel_worker_start', False)
self.log.info(
'{bootmsg} {method}',
bootmsg=hl(
'Booting node from local configuration [parallel_worker_start={}] ..'
.format(parallel_worker_start),
color='green',
bold=True),
method=hltype(Node.boot_from_config))
# start Manhole in node controller
if 'manhole' in controller:
yield self._controller.call(u'crossbar.start_manhole',
controller['manhole'],
options=CallOptions())
self.log.debug("controller: manhole started")
# startup all workers
workers = config.get('workers', [])
if len(workers):
self.log.info(
hl('Will start {} worker{} ..'.format(
len(workers), 's' if len(workers) > 1 else ''),
color='green',
bold=True))
else:
self.log.info(
hl('No workers configured, nothing to do',
color='green',
bold=True))
dl = []
for worker in workers:
# worker ID
if 'id' in worker:
worker_id = worker['id']
else:
worker_id = u'worker{:03d}'.format(self._worker_no)
worker['id'] = worker_id
self._worker_no += 1
# worker type: either a native worker ('router', 'container', ..), or a guest worker ('guest')
worker_type = worker['type']
# native worker processes setup
if worker_type in self._native_workers:
# set logname depending on native worker type
worker_logname = '{} {}'.format(
self._native_workers[worker_type]['logname'],
hlid(worker_id))
# any worker specific options
worker_options = worker.get('options', {})
# start the (native) worker
self.log.info(
"Order node to start {worker_logname}",
worker_logname=worker_logname,
)
d = self._controller.call(u'crossbar.start_worker',
worker_id,
worker_type,
worker_options,
options=CallOptions())
@inlineCallbacks
def configure_worker(res, worker_logname, worker_type,
worker_id, worker):
self.log.info(
"Ok, node has started {worker_logname}",
worker_logname=worker_logname,
)
# now configure the worker
self.log.info(
"Configuring {worker_logname} ..",
worker_logname=worker_logname,
)
method_name = '_configure_native_worker_{}'.format(
worker_type.replace('-', '_'))
try:
config_fn = getattr(self, method_name)
except AttributeError:
raise ValueError(
"A native worker of type '{}' is configured but "
"there is no method '{}' on {}".format(
worker_type, method_name, type(self)))
yield config_fn(worker_logname, worker_id, worker)
self.log.info(
"Ok, {worker_logname} configured",
worker_logname=worker_logname,
)
d.addCallback(configure_worker, worker_logname, worker_type,
worker_id, worker)
# guest worker processes setup
elif worker_type == u'guest':
# now actually start the (guest) worker ..
# FIXME: start_worker() takes the whole configuration item for guest workers, whereas native workers
# only take the options (which is part of the whole config item for the worker)
d = self._controller.call(u'crossbar.start_worker',
worker_id,
worker_type,
worker,
options=CallOptions())
else:
raise Exception(
'logic error: unexpected worker_type="{}"'.format(
worker_type))
if parallel_worker_start:
dl.append(d)
else:
yield d
yield gatherResults(dl)
self.log.info(
hl('Ok, local node configuration booted successfully!',
color='green',
bold=True))
def _configure_native_worker_proxy(self, worker_logname, worker_id, worker):
yield self._configure_native_worker_common(worker_logname, worker_id, worker)
# start transports on proxy
for i, transport in enumerate(worker.get('transports', [])):
if 'id' in transport:
transport_id = transport['id']
else:
transport_id = 'transport{:03d}'.format(i)
transport['id'] = transport_id
self.log.info(
"Order {worker_logname} to start Transport {transport_id}",
worker_logname=worker_logname,
transport_id=hlid(transport_id),
)
# XXX we're doing startup, and begining proxy workers --
# want to share the web-transport etc etc stuff between
# these and otehr kinds of routers / transports
yield self._controller.call('crossbar.worker.{}.start_proxy_transport'.format(worker_id),
transport_id,
transport,
options=CallOptions())
if transport['type'] == 'web':
paths = transport.get('paths', {})
elif transport['type'] in ('universal'):
paths = transport.get('web', {}).get('paths', {})
else:
paths = None
# Web service paths
if paths:
for path in sorted(paths):
if path != '/':
webservice = paths[path]
if 'id' in webservice:
webservice_id = webservice['id']
else:
webservice_id = 'webservice{:03d}'.format(self._webservice_no)
webservice['id'] = webservice_id
self._webservice_no += 1
self.log.info(
"Order Transport {transport_id} to start Web Service {webservice_id}",
transport_id=hlid(transport_id),
webservice_id=hlid(webservice_id),
path=hluserid(path),
)
yield self._controller.call('crossbar.worker.{}.start_web_transport_service'.format(worker_id),
transport_id,
path,
webservice,
options=CallOptions())
self.log.info(
"Ok, Transport {transport_id} has started Web Service {webservice_id}",
transport_id=hlid(transport_id),
webservice_id=hlid(webservice_id),
path=hluserid(path),
)
self.log.info(
"Ok, {worker_logname} has started Transport {transport_id}",
worker_logname=worker_logname,
transport_id=hlid(transport_id),
)
# set up backend connections on the proxy
for i, connection_name in enumerate(worker.get('connections', {})):
self.log.debug(
"Starting connection {index}: {name}",
index=i,
name=connection_name,
)
yield self._controller.call(
'crossbar.worker.{}.start_proxy_connection'.format(worker_id),
connection_name,
worker['connections'].get(connection_name, {}),
)
# set up realms and roles on the proxy
for i, realm_name in enumerate(worker.get('routes', {})):
roles = worker['routes'][realm_name]
for role_id, connections in roles.items():
if not isinstance(connections, list):
connections = [connections] # used to be a single string, now a list of strings
for connection_id in connections:
self.log.debug(
"Starting proxy realm route {realm}, {role} to {connection}",
realm=realm_name,
role=role_id,
connection=connection_id,
)
yield self._controller.call(
'crossbar.worker.{}.start_proxy_realm_route'.format(worker_id),
realm_name,
{role_id: connection_id},
)
def _configure_native_worker_router(self, worker_logname, worker_id, worker):
yield self._configure_native_worker_common(worker_logname, worker_id, worker)
# start realms on router
for realm in worker.get('realms', []):
# start realm
if 'id' in realm:
realm_id = realm['id']
else:
realm_id = 'realm{:03d}'.format(self._realm_no)
realm['id'] = realm_id
self._realm_no += 1
self.log.info(
"Order {worker_logname} to start Realm {realm_id}",
worker_logname=worker_logname,
realm_id=hlid(realm_id),
)
yield self._controller.call('crossbar.worker.{}.start_router_realm'.format(worker_id),
realm_id,
realm,
options=CallOptions())
self.log.info(
"Ok, {worker_logname} has started Realm {realm_id}",
worker_logname=worker_logname,
realm_id=hlid(realm_id),
)
# add roles to realm
for role in realm.get('roles', []):
if 'id' in role:
role_id = role['id']
else:
role_id = 'role{:03d}'.format(self._role_no)
role['id'] = role_id
self._role_no += 1
self.log.info(
"Order Realm {realm_id} to start Role {role_id}",
realm_id=hlid(realm_id),
role_id=hlid(role_id),
)
yield self._controller.call('crossbar.worker.{}.start_router_realm_role'.format(worker_id),
realm_id,
role_id,
role,
options=CallOptions())
self.log.info(
"Ok, Realm {realm_id} has started Role {role_id}",
realm_id=hlid(realm_id),
role_id=hlid(role_id),
)
# start components to run embedded in the router
for component in worker.get('components', []):
if 'id' in component:
component_id = component['id']
else:
component_id = 'component{:03d}'.format(self._component_no)
component['id'] = component_id
self._component_no += 1
yield self._controller.call('crossbar.worker.{}.start_router_component'.format(worker_id),
component_id,
component,
options=CallOptions())
self.log.info(
"{logname}: component '{component}' started",
logname=worker_logname,
component=component_id,
)
# start transports on router
for transport in worker.get('transports', []):
if 'id' in transport:
transport_id = transport['id']
else:
transport_id = 'transport{:03d}'.format(self._transport_no)
transport['id'] = transport_id
self._transport_no += 1
add_paths_on_transport_create = False
self.log.info(
"Order {worker_logname} to start Transport {transport_id}",
worker_logname=worker_logname,
transport_id=hlid(transport_id),
)
yield self._controller.call('crossbar.worker.{}.start_router_transport'.format(worker_id),
transport_id,
transport,
create_paths=add_paths_on_transport_create,
options=CallOptions())
self.log.info(
"Ok, {worker_logname} has started Transport {transport_id}",
worker_logname=worker_logname,
transport_id=hlid(transport_id),
)
if not add_paths_on_transport_create:
if transport['type'] == 'web':
paths = transport.get('paths', {})
elif transport['type'] in ('universal'):
paths = transport.get('web', {}).get('paths', {})
else:
paths = None
# Web service paths
if paths:
for path in sorted(paths):
if path != '/':
webservice = paths[path]
if 'id' in webservice:
webservice_id = webservice['id']
else:
webservice_id = 'webservice{:03d}'.format(self._webservice_no)
webservice['id'] = webservice_id
self._webservice_no += 1
self.log.info(
"Order Transport {transport_id} to start Web Service {webservice_id}",
transport_id=hlid(transport_id),
webservice_id=hlid(webservice_id),
path=hluserid(path),
)
yield self._controller.call(
'crossbar.worker.{}.start_web_transport_service'.format(worker_id),
transport_id,
path,
webservice,
options=CallOptions())
self.log.info(
"Ok, Transport {transport_id} has started Web Service {webservice_id}",
transport_id=hlid(transport_id),
webservice_id=hlid(webservice_id),
path=hluserid(path),
)
# start rlinks for realms
dl = []
for realm in worker.get('realms', []):
realm_id = realm['id']
for i, rlink in enumerate(realm.get('rlinks', [])):
if 'id' in rlink:
rlink_id = rlink['id']
else:
rlink_id = 'rlink{:03d}'.format(i)
rlink['id'] = rlink_id
self.log.info(
'Starting router-to-router "{rlink_id}" on realm "{realm_id}" ..',
realm_id=hlid(realm_id),
rlink_id=hlid(rlink_id),
)
d = self._controller.call('crossbar.worker.{}.start_router_realm_link'.format(worker_id),
realm_id,
rlink_id,
rlink,
options=CallOptions())
def done(_):
self.log.info(
'Ok, router-to-router {rlink_id} started on realm "{realm_id}".',
realm_id=hlid(realm_id),
rlink_id=hlid(rlink_id),
)
d.addCallback(done)
dl.append(d)
# FIXME: rlinks must be started without waiting for them to be established. otherwise the start of other stuff
# is waiting for all rlinks to be up!
d = gatherResults(dl)
