def run(): def on_progress(data): print(f'on_progress: {len(data)}') global progress_file progress_file += data['data'] if progressive: print(f'Progressive call to method {uri} with {_args}, {_kwargs}') options = CallOptions(on_progress=on_progress) else: print(f"call method {uri} with {_args}, {_kwargs}") options = CallOptions() try: ret = yield session.call(uri, *_args, **_kwargs, options=options) except Exception as e: print(e) else: try: ret = json.dumps(ret, indent=4) except Exception as e: pass else: print(ret) if tofile: with open(tofile, 'w') as f: f.write(ret) global progress_file if progress_file: with open(tofile, 'wb') as f: data = base64.b64decode(progress_file) f.write(data) session.leave()
def _configure_native_worker_common(self, worker_logname, worker_id, worker): # expanding PYTHONPATH of the newly started worker is now done # directly in NodeController._start_native_worker worker_options = worker.get('options', {}) if False: if 'pythonpath' in worker_options: added_paths = yield self._controller.call( u'crossbar.worker.{}.add_pythonpath'.format(worker_id), worker_options['pythonpath'], options=CallOptions()) self.log.warn("{worker}: PYTHONPATH extended for {paths}", worker=worker_logname, paths=added_paths) # FIXME: as the CPU affinity is in the worker options, this _also_ (see above fix) # should be done directly in NodeController._start_native_worker if True: if 'cpu_affinity' in worker_options: new_affinity = yield self._controller.call( u'crossbar.worker.{}.set_cpu_affinity'.format(worker_id), worker_options['cpu_affinity'], options=CallOptions()) self.log.debug("{worker}: CPU affinity set to {affinity}", worker=worker_logname, affinity=new_affinity) # this is fine to start after the worker has been started, as manhole is # CB developer/support feature anyways (like a vendor diagnostics port) if 'manhole' in worker: yield self._controller.call( u'crossbar.worker.{}.start_manhole'.format(worker_id), worker['manhole'], options=CallOptions()) self.log.debug("{worker}: manhole started", worker=worker_logname)
def __init__(self, client): self.client = client self.NeuroArchWrite_rpc = partial(client.rpc, 'ffbo.na.NeuroArch.write.{}'.format(client.naServerID), options=CallOptions(timeout=10000)) self.NeuroArchQuery_rpc = partial(client.rpc, 'ffbo.na.NeuroArch.query.{}'.format(client.naServerID), options=CallOptions(timeout=10000))
def _configure_native_worker_proxy(self, worker_logname, worker_id, worker): yield self._configure_native_worker_common(worker_logname, worker_id, worker) # start transports on proxy for i, transport in enumerate(worker.get('transports', [])): if 'id' in transport: transport_id = transport['id'] else: transport_id = 'transport{:03d}'.format(i) transport['id'] = transport_id self.log.info( "Order {worker_logname} to start Transport {transport_id}", worker_logname=worker_logname, transport_id=hlid(transport_id), ) # XXX we're doing startup, and begining proxy workers -- # want to share the web-transport etc etc stuff between # these and otehr kinds of routers / transports yield self._controller.call(u'crossbar.worker.{}.start_proxy_transport'.format(worker_id), transport_id, transport, options=CallOptions()) self.log.info( "Ok, {worker_logname} has started Transport {transport_id}", worker_logname=worker_logname, transport_id=hlid(transport_id), ) for i, backend in enumerate(worker.get('backends', [])): if 'id' in backend: backend_id = backend['id'] else: backend_id = 'backend{:03d}'.format(i) backend['id'] = backend_id self.log.info( "Order {worker_logname} to start BackendTransport {backend_id}", worker_logname=worker_logname, backend_id=hlid(backend_id), ) yield self._controller.call(u'crossbar.worker.{}.start_proxy_backend'.format(worker_id), backend_id, backend, options=CallOptions()) self.log.info( "Ok, {worker_logname} has started BackendTransport {backend_id}", worker_logname=worker_logname, transport_id=hlid(backend_id), )
def onJoin(self, details): print("session joined") # ---- Publisher ---- for i in range(5): msg = u'Testing %d' % i print("Publishing:", msg) self.publish(topic, msg) yield sleep(1) # ---- Caller ---- print('Calling procedure:', procedure) try: res = yield self.call(procedure, 2, 3) print("call result: {}".format(res)) except Exception as e: print("call error: {0}".format(e)) def on_prog(msg): print("here") print("partial result: {}".format(msg)) print('Calling procedure:', procedure2) try: res = yield self.call(procedure2, 'A', options=CallOptions(on_progress=on_prog)) print("call result: {}".format(res)) except Exception as e: print("call error: {0}".format(e))
def on_call(*args, **kwargs): assert 'details' in kwargs details = kwargs.pop('details') options = kwargs.pop('options', None) if details.caller is None or details.caller_authrole is None or details.caller_authid is None: raise RuntimeError( "Internal error attempting rlink forwarding" ) self.log.info( 'Received invocation on uri={uri}, options={options} (caller={caller}, caller_authid={caller_authid}, caller_authrole={caller_authrole}, forward_for={forward_for})', uri=uri, options=options, caller=details.caller, caller_authid=details.caller_authid, caller_authrole=details.caller_authrole, forward_for=details.forward_for) this_forward = { 'session': details.caller, 'authid': details.caller_authrole, 'authrole': details.caller_authrole, } if details.forward_for: # the call comes already forwarded from a router node .. if len(details.forward_for) >= 0: self.log.debug('SKIP! already forwarded') return forward_for = copy.deepcopy(details.forward_for) forward_for.append(this_forward) else: forward_for = [this_forward] options = CallOptions(forward_for=forward_for) try: result = yield self.call(uri, *args, options=options, **kwargs) except TransportLost: return except ApplicationError as e: if e.error not in ['wamp.close.normal']: self.log.warn('FAILED TO CALL 1: {} {}'.format(type(e), str(e))) return except Exception as e: if not ERR_MSG[0]: self.log.warn('FAILED TO CALL 2: {} {}'.format(type(e), str(e))) ERR_MSG[0] = True return self.log.info( "RLink forward-invoked call {dir} (options={options})", dir=self.DIR, options=options, ) return result
async def call(self, endpoint: str, *args, on_progress: Callable[..., None] = None, timeout: int = None, **kwargs): """ Call an RPC function. :param endpoint: name of the endpoint to call :param args: positional arguments to call the endpoint with :param on_progress: a callable that will receive progress reports from the endpoint :param timeout: timeout (in seconds) to wait for the completion of the call :param kwargs: keyword arguments to call the endpoint with :return: the return value of the call :raises TimeoutError: if the call times out """ assert check_argument_types() if self._session is None: await self.connect() options = CallOptions(on_progress=on_progress, timeout=timeout) return await self._session.call(endpoint, *args, options=options, **kwargs)
def __init__(self, config=None): ApplicationSession.__init__(self, config) global live live = self self.logger = logging.getLogger('Live') self.logger.info("Config: %s", config) self.account_id = config.extra['authid'] self.secret = config.extra['secret'] if '-' not in self.account_id: self.account_id = "local-%s" % self.account_id self.authid = '%s:%s' % (self.account_id, self.secret[-7:]) self.joined = False self.lock = DeferredLock() self.checks = {} self.workers = {} self.CallOptions = CallOptions()
def call_progress(topic): print("calling '{}' progressively".format(topic)) def on_progress(some_data): print("received: '{}'".format(some_data)) return session.call(topic, options=CallOptions(on_progress=on_progress))
def select_DataSource(self, name, version = None): uri = "ffbo.na.datasource.{}".format(self.client.naServerID) res = self.client.rpc( uri, name, version = version, options=CallOptions(timeout=10000) ) result = self._check_result(res) return result
def executeNAquery(self, res, language='en', uri=None, queryID=None, progressive=True, threshold=5): """Execute an NA query. Args: res (dict): Neuroarch query. language (str): Language to use. uri (str): A custom FFBO query URI if desired. queryID (str): Query ID to be used. Generated automatically. progressive (bool): Whether the loading should be progressive. Needs to be true most of the time for connection to be stable. threshold (int): Data chunk size. Low threshold is required for the connection to be stable. Returns: bool: Whether the process has been successful. """ def on_progress(x, res): res.append(x) if isinstance(res, str): res = json.loads(res) if uri == None: uri = 'ffbo.na.query.' + self.naServerID if "uri" in res.keys(): uri = res["uri"] + "." + self.naServerID if queryID == None: queryID = guidGenerator() del self.data # Reset the data in the backend self.data = [] res['queryID'] = queryID res['threshold'] = threshold res['data_callback_uri'] = 'ffbo.ui.receive_data' res_list = [] res = self.client.session.call( uri, res, options=CallOptions( on_progress=partial(on_progress, res=res_list))) a = {} a['data'] = res a['messageType'] = 'Data' a['widget'] = 'NLP' if "retrieve_tag" in uri: a['messageType'] = 'TagData' self.tryComms(a) self.executeNAquery({"command": {"retrieve": {"state": 0}}}) if progressive == True: self.tryComms(a) self.data.append(a) return self.data else: self.tryComms(a) return a
def onJoin(self, details): def on_progress(i): print("Progress: {}".format(i)) res = yield from self.call('com.myapp.longop', 3, options=CallOptions(onProgress=on_progress)) print("Final: {}".format(res)) self.leave()
def onJoin(self, details): print("session attached") def on_progress(i): print("Progress: {}".format(i)) res = yield self.call(u'com.myapp.longop', 3, options=CallOptions(on_progress=on_progress)) print("Final: {}".format(res)) self.leave()
def onJoin(self, details): def on_event(val): print("Someone requested to square non-positive: {}".format(val)) yield from self.subscribe(on_event, 'com.myapp.square_on_nonpositive') for val in [2, 0, -2]: res = yield from self.call('com.myapp.square', val, options=CallOptions(disclose_me=True)) print("Squared {} = {}".format(val, res)) yield from self.leave()
async def onJoin(self, details): def on_event(val): print("Someone requested to square non-positive: {}".format(val)) await self.subscribe(on_event, u'com.myapp.square_on_nonpositive') for val in [2, 0, -2]: res = await self.call(u'com.myapp.square', val, options=CallOptions()) print("Squared {} = {}".format(val, res)) await self.leave()
def _test_rpc_errors(self): options = CallOptions() for proc in [(u'com.myapp.failme', False), (u'com.myapp.failme', True), (u'com.myapp.encrypted.failme', False), (u'com.myapp.encrypted.failme', True)]: try: res = yield self.call(proc[0], proc[1], options=options) print("{} call with {} - result: {}".format( proc[0], proc[1], res)) except Exception as e: print("{} call with {} - error: {}".format( proc[0], proc[1], e))
def onJoin(self, details): print("session attached") def on_event(val): print("Someone requested to square non-positive: {}".format(val)) yield self.subscribe(on_event, 'com.myapp.square_on_nonpositive') for val in [2, 0, -2]: res = yield self.call('com.myapp.square', val, options=CallOptions()) print("Squared {} = {}".format(val, res)) self.leave()
def _test_rpc(self, delay=None): options = CallOptions() counter = 1 while counter <= self.NUM: for proc in [u'com.myapp.add2', u'com.myapp.encrypted.add2']: try: res = yield self.call(proc, 23, counter, options=options) print("{} call result: {}".format(proc, res)) except Exception as e: print("{} call error: {}".format(proc, e)) if delay: yield sleep(delay) counter += 1
def _configure_native_worker_container(self, worker_logname, worker_id, worker): yield self._configure_native_worker_common(worker_logname, worker_id, worker) # if components exit "very soon after" we try to start them, # we consider that a failure and shut our node down. We remove # this subscription 2 seconds after we're done starting # everything (see below). This is necessary as start_component # returns as soon as we've established a connection to the # component def component_exited(info): component_id = info.get("id") self.log.critical( "Component '{component_id}' failed to start; shutting down node.", component_id=component_id) try: self._reactor.stop() except twisted.internet.error.ReactorNotRunning: pass topic = u'crossbar.worker.{}.container.on_component_stop'.format( worker_id) component_stop_sub = yield self._controller.subscribe( component_exited, topic) # start components to run embedded in the container # for component in worker.get('components', []): if 'id' in component: component_id = component['id'] else: component_id = 'component{:03d}'.format(self._component_no) component['id'] = component_id self._component_no += 1 yield self._controller.call( u'crossbar.worker.{}.start_component'.format(worker_id), component_id, component, options=CallOptions()) self.log.info("{worker}: component '{component_id}' started", worker=worker_logname, component_id=component_id) # after 2 seconds, consider all the application components running self._reactor.callLater(2, component_stop_sub.unsubscribe)
def neuroarch_query(request, details=None): """ Call the neuroarch_query module with a neuroarch-json object The request should have user: session_id for client server: session_id for na server query: neuroarch json query object """ try: request['user'] = details.caller user_details = yield self.call( six.u('ffbo.auth_server.get_user'), details.caller) if user_details: request['username'] = user_details['username'] self.log.info( "neuroarch_query() accessed with request: {request}", request=request) progressive_result = {} def on_progress(p): progressive_result.update(p) #print request result = yield self.call( six.u('ffbo.na.query.' + str(request['server'])), request, options=CallOptions(on_progress=on_progress)) self.log.info("na_query returned with result") if progressive_result: result = progressive_result # Catch no results and return if result == "{}": raise ValueError('Neuroarch returned zero results') returnValue(feedback_success(request, (result))) except ValueError as e: feedback = feedback_error(request, "Neuroarch returned zero results", e) returnValue(feedback) except ApplicationError as e: raise e feedback = feedback_error(server_id, "Unable to contact server", e) returnValue(feedback)
def _test_rpc(self, delay=None): options = CallOptions() counter = 1 while counter <= self.NUM: for proc in ['com.myapp.add2', 'com.myapp.encrypted.add2']: try: res = yield self.call(proc, 23, counter, options=options) self.log.info('{proc} call result: {res}', proc=proc, res=res) except Exception as e: self.log.info('{proc} call error: {e}', proc=proc, e=e) if delay: yield sleep(delay) counter += 1
async def worker(index, cid): class Data(): def __init__(self): self.data = bytes() def progress(self, update): self.data += bytes(update) #get the binary data uri = f"ocp.documents.{self.onlineDoc.id}.raw.BinaryByCid" dat = Data() opt = CallOptions(on_progress=dat.progress) result = await self.onlineDoc.connection.api.call( uri, cid, options=opt) if result is not None: dat.progress(result) values[index] = dat.data
def _configure_native_worker_websocket_testee(self, worker_logname, worker_id, worker): yield self._configure_native_worker_common(worker_logname, worker_id, worker) # start transport on websocket-testee transport = worker['transport'] transport_id = 'transport{:03d}'.format(self._transport_no) transport['id'] = transport_id self._transport_no = 1 yield self._controller.call('crossbar.worker.{}.start_websocket_testee_transport'.format(worker_id), transport_id, transport, options=CallOptions()) self.log.info( "{logname}: transport '{tid}' started", logname=worker_logname, tid=transport_id, )
def _test_rpc_errors(self): options = CallOptions() for proc, encrypted_error in [('com.myapp.failme', False), ('com.myapp.failme', True), ('com.myapp.encrypted.failme', False), ('com.myapp.encrypted.failme', True)]: try: res = yield self.call(proc, encrypted_error, options=options) self.log.info( '{proc} called with encrypted_error={encrypted_error} - result: {res}', proc=proc, encrypted_error=encrypted_error, res=res) except Exception as e: self.log.info( '{proc} called with encrypted_error={encrypted_error} - error: {e}', proc=proc, encrypted_error=encrypted_error, e=e)
def _configure_native_worker_container(self, worker_logname, worker_id, worker): yield self._configure_native_worker_common(worker_logname, worker_id, worker) # start components to run embedded in the container # for component in worker.get('components', []): if 'id' in component: component_id = component['id'] else: component_id = 'component{:03d}'.format(self._component_no) component['id'] = component_id self._component_no += 1 yield self._controller.call('crossbar.worker.{}.start_component'.format(worker_id), component_id, component, options=CallOptions()) self.log.info("{worker}: component '{component_id}' started", worker=worker_logname, component_id=component_id)
def _startup(self, config): """ Startup elements in the node as specified in the provided node configuration. """ self.log.info('Configuring node from local configuration ...') # call options we use to call into the local node management API call_options = CallOptions() # fake call details we use to call into the local node management API call_details = CallDetails(caller=0) # get contoller configuration subpart controller = config.get('controller', {}) # start Manhole in node controller if 'manhole' in controller: yield self._controller.start_manhole(controller['manhole'], details=call_details) # startup all workers workers = config.get('workers', []) if len(workers): self.log.info('Starting {nworkers} workers ...', nworkers=len(workers)) else: self.log.info('No workers configured!') for worker in workers: # worker ID if 'id' in worker: worker_id = worker.pop('id') else: worker_id = 'worker-{:03d}'.format(self._worker_no) self._worker_no += 1 # worker type - a type of working process from the following fixed list worker_type = worker['type'] assert (worker_type in ['router', 'container', 'guest', 'websocket-testee']) # set logname depending on worker type if worker_type == 'router': worker_logname = "Router '{}'".format(worker_id) elif worker_type == 'container': worker_logname = "Container '{}'".format(worker_id) elif worker_type == 'websocket-testee': worker_logname = "WebSocketTestee '{}'".format(worker_id) elif worker_type == 'guest': worker_logname = "Guest '{}'".format(worker_id) else: raise Exception("logic error") # any worker specific options worker_options = worker.get('options', {}) # native worker processes: router, container, websocket-testee if worker_type in ['router', 'container', 'websocket-testee']: # start a new native worker process .. if worker_type == 'router': yield self._controller.start_router(worker_id, worker_options, details=call_details) elif worker_type == 'container': yield self._controller.start_container( worker_id, worker_options, details=call_details) elif worker_type == 'websocket-testee': yield self._controller.start_websocket_testee( worker_id, worker_options, details=call_details) else: raise Exception("logic error") # setup native worker generic stuff if 'pythonpath' in worker_options: added_paths = yield self._controller.call( 'crossbar.worker.{}.add_pythonpath'.format(worker_id), worker_options['pythonpath'], options=call_options) self.log.debug("{worker}: PYTHONPATH extended for {paths}", worker=worker_logname, paths=added_paths) if 'cpu_affinity' in worker_options: new_affinity = yield self._controller.call( 'crossbar.worker.{}.set_cpu_affinity'.format( worker_id), worker_options['cpu_affinity'], options=call_options) self.log.debug("{worker}: CPU affinity set to {affinity}", worker=worker_logname, affinity=new_affinity) if 'manhole' in worker: yield self._controller.call( 'crossbar.worker.{}.start_manhole'.format(worker_id), worker['manhole'], options=call_options) self.log.debug("{worker}: manhole started", worker=worker_logname) # setup router worker if worker_type == 'router': # start realms on router for realm in worker.get('realms', []): # start realm if 'id' in realm: realm_id = realm.pop('id') else: realm_id = 'realm-{:03d}'.format(self._realm_no) self._realm_no += 1 yield self._controller.call( 'crossbar.worker.{}.start_router_realm'.format( worker_id), realm_id, realm, options=call_options) self.log.info( "{worker}: realm '{realm_id}' (named '{realm_name}') started", worker=worker_logname, realm_id=realm_id, realm_name=realm['name']) # add roles to realm for role in realm.get('roles', []): if 'id' in role: role_id = role.pop('id') else: role_id = 'role-{:03d}'.format(self._role_no) self._role_no += 1 yield self._controller.call( 'crossbar.worker.{}.start_router_realm_role'. format(worker_id), realm_id, role_id, role, options=call_options) self.log.info( "{logname}: role '{role}' (named '{role_name}') started on realm '{realm}'", logname=worker_logname, role=role_id, role_name=role['name'], realm=realm_id, ) # start uplinks for realm for uplink in realm.get('uplinks', []): if 'id' in uplink: uplink_id = uplink.pop('id') else: uplink_id = 'uplink-{:03d}'.format( self._uplink_no) self._uplink_no += 1 yield self._controller.call( 'crossbar.worker.{}.start_router_realm_uplink'. format(worker_id), realm_id, uplink_id, uplink, options=call_options) self.log.info( "{logname}: uplink '{uplink}' started on realm '{realm}'", logname=worker_logname, uplink=uplink_id, realm=realm_id, ) # start connections (such as PostgreSQL database connection pools) # to run embedded in the router for connection in worker.get('connections', []): if 'id' in connection: connection_id = connection.pop('id') else: connection_id = 'connection-{:03d}'.format( self._connection_no) self._connection_no += 1 yield self._controller.call( 'crossbar.worker.{}.start_connection'.format( worker_id), connection_id, connection, options=call_options) self.log.info( "{logname}: connection '{connection}' started", logname=worker_logname, connection=connection_id, ) # start components to run embedded in the router for component in worker.get('components', []): if 'id' in component: component_id = component.pop('id') else: component_id = 'component-{:03d}'.format( self._component_no) self._component_no += 1 yield self._controller.call( 'crossbar.worker.{}.start_router_component'.format( worker_id), component_id, component, options=call_options) self.log.info( "{logname}: component '{component}' started", logname=worker_logname, component=component_id, ) # start transports on router for transport in worker['transports']: if 'id' in transport: transport_id = transport.pop('id') else: transport_id = 'transport-{:03d}'.format( self._transport_no) self._transport_no += 1 yield self._controller.call( 'crossbar.worker.{}.start_router_transport'.format( worker_id), transport_id, transport, options=call_options) self.log.info( "{logname}: transport '{tid}' started", logname=worker_logname, tid=transport_id, ) # setup container worker elif worker_type == 'container': # if components exit "very soon after" we try to # start them, we consider that a failure and shut # our node down. We remove this subscription 2 # seconds after we're done starting everything # (see below). This is necessary as # start_container_component returns as soon as # we've established a connection to the component def component_exited(info): component_id = info.get("id") self.log.critical( "Component '{component_id}' failed to start; shutting down node.", component_id=component_id) try: self._reactor.stop() except twisted.internet.error.ReactorNotRunning: pass topic = 'crossbar.worker.{}.container.on_component_stop'.format( worker_id) component_stop_sub = yield self._controller.subscribe( component_exited, topic) # start connections (such as PostgreSQL database connection pools) # to run embedded in the container # for connection in worker.get('connections', []): if 'id' in connection: connection_id = connection.pop('id') else: connection_id = 'connection-{:03d}'.format( self._connection_no) self._connection_no += 1 yield self._controller.call( 'crossbar.worker.{}.start_connection'.format( worker_id), connection_id, connection, options=call_options) self.log.info( "{logname}: connection '{connection}' started", logname=worker_logname, connection=connection_id, ) # start components to run embedded in the container # for component in worker.get('components', []): if 'id' in component: component_id = component.pop('id') else: component_id = 'component-{:03d}'.format( self._component_no) self._component_no += 1 yield self._controller.call( 'crossbar.worker.{}.start_container_component'. format(worker_id), component_id, component, options=call_options) self.log.info( "{worker}: component '{component_id}' started", worker=worker_logname, component_id=component_id) # after 2 seconds, consider all the application components running self._reactor.callLater(2, component_stop_sub.unsubscribe) # setup websocket-testee worker elif worker_type == 'websocket-testee': # start transport on websocket-testee transport = worker['transport'] transport_id = 'transport-{:03d}'.format( self._transport_no) self._transport_no = 1 yield self._controller.call( 'crossbar.worker.{}.start_websocket_testee_transport'. format(worker_id), transport_id, transport, options=call_options) self.log.info( "{logname}: transport '{tid}' started", logname=worker_logname, tid=transport_id, ) else: raise Exception("logic error") elif worker_type == 'guest': # start guest worker # yield self._controller.start_guest(worker_id, worker, details=call_details) self.log.info("{worker}: started", worker=worker_logname) else: raise Exception("logic error") self.log.info('Local node configuration applied.')
def _configure_native_worker_router(self, worker_logname, worker_id, worker): yield self._configure_native_worker_common(worker_logname, worker_id, worker) # start realms on router for realm in worker.get('realms', []): # start realm if 'id' in realm: realm_id = realm['id'] else: realm_id = 'realm{:03d}'.format(self._realm_no) realm['id'] = realm_id self._realm_no += 1 self.log.info( "Order {worker_logname} to start Realm {realm_id}", worker_logname=worker_logname, realm_id=hlid(realm_id), ) yield self._controller.call( u'crossbar.worker.{}.start_router_realm'.format(worker_id), realm_id, realm, options=CallOptions()) self.log.info( "Ok, {worker_logname} has started Realm {realm_id}", worker_logname=worker_logname, realm_id=hlid(realm_id), ) # add roles to realm for role in realm.get('roles', []): if 'id' in role: role_id = role['id'] else: role_id = 'role{:03d}'.format(self._role_no) role['id'] = role_id self._role_no += 1 self.log.info( "Order Realm {realm_id} to start Role {role_id}", realm_id=hlid(realm_id), role_id=hlid(role_id), ) yield self._controller.call( u'crossbar.worker.{}.start_router_realm_role'.format( worker_id), realm_id, role_id, role, options=CallOptions()) self.log.info( "Ok, Realm {realm_id} has started Role {role_id}", realm_id=hlid(realm_id), role_id=hlid(role_id), ) # start components to run embedded in the router for component in worker.get('components', []): if 'id' in component: component_id = component['id'] else: component_id = 'component{:03d}'.format(self._component_no) component['id'] = component_id self._component_no += 1 yield self._controller.call( u'crossbar.worker.{}.start_router_component'.format(worker_id), component_id, component, options=CallOptions()) self.log.info( "{logname}: component '{component}' started", logname=worker_logname, component=component_id, ) # start transports on router for transport in worker.get('transports', []): if 'id' in transport: transport_id = transport['id'] else: transport_id = 'transport{:03d}'.format(self._transport_no) transport['id'] = transport_id self._transport_no += 1 add_paths_on_transport_create = False self.log.info( "Order {worker_logname} to start Transport {transport_id}", worker_logname=worker_logname, transport_id=hlid(transport_id), ) yield self._controller.call( u'crossbar.worker.{}.start_router_transport'.format(worker_id), transport_id, transport, create_paths=add_paths_on_transport_create, options=CallOptions()) self.log.info( "Ok, {worker_logname} has started Transport {transport_id}", worker_logname=worker_logname, transport_id=hlid(transport_id), ) if not add_paths_on_transport_create: if transport['type'] == 'web': paths = transport.get('paths', {}) elif transport['type'] == 'universal': paths = transport.get('web', {}).get('paths', {}) else: paths = None # Web service paths if paths: for path in sorted(paths): if path != '/': webservice = paths[path] if 'id' in webservice: webservice_id = webservice['id'] else: webservice_id = 'webservice{:03d}'.format( self._webservice_no) webservice['id'] = webservice_id self._webservice_no += 1 self.log.info( "Order Transport {transport_id} to start Web Service {webservice_id}", transport_id=hlid(transport_id), webservice_id=hlid(webservice_id), path=hluserid(path), ) yield self._controller.call( u'crossbar.worker.{}.start_web_transport_service' .format(worker_id), transport_id, path, webservice, options=CallOptions()) self.log.info( "Ok, Transport {transport_id} has started Web Service {webservice_id}", transport_id=hlid(transport_id), webservice_id=hlid(webservice_id), path=hluserid(path), )
def boot_from_config(self, config): """ Startup elements in the node as specified in the provided node configuration. """ # get controller configuration subpart controller = config.get('controller', {}) parallel_worker_start = controller.get('options', {}).get( 'enable_parallel_worker_start', False) self.log.info( '{bootmsg} {method}', bootmsg=hl( 'Booting node from local configuration [parallel_worker_start={}] ..' .format(parallel_worker_start), color='green', bold=True), method=hltype(Node.boot_from_config)) # start Manhole in node controller if 'manhole' in controller: yield self._controller.call(u'crossbar.start_manhole', controller['manhole'], options=CallOptions()) self.log.debug("controller: manhole started") # startup all workers workers = config.get('workers', []) if len(workers): self.log.info( hl('Will start {} worker{} ..'.format( len(workers), 's' if len(workers) > 1 else ''), color='green', bold=True)) else: self.log.info( hl('No workers configured, nothing to do', color='green', bold=True)) dl = [] for worker in workers: # worker ID if 'id' in worker: worker_id = worker['id'] else: worker_id = u'worker{:03d}'.format(self._worker_no) worker['id'] = worker_id self._worker_no += 1 # worker type: either a native worker ('router', 'container', ..), or a guest worker ('guest') worker_type = worker['type'] # native worker processes setup if worker_type in self._native_workers: # set logname depending on native worker type worker_logname = '{} {}'.format( self._native_workers[worker_type]['logname'], hlid(worker_id)) # any worker specific options worker_options = worker.get('options', {}) # start the (native) worker self.log.info( "Order node to start {worker_logname}", worker_logname=worker_logname, ) d = self._controller.call(u'crossbar.start_worker', worker_id, worker_type, worker_options, options=CallOptions()) @inlineCallbacks def configure_worker(res, worker_logname, worker_type, worker_id, worker): self.log.info( "Ok, node has started {worker_logname}", worker_logname=worker_logname, ) # now configure the worker self.log.info( "Configuring {worker_logname} ..", worker_logname=worker_logname, ) method_name = '_configure_native_worker_{}'.format( worker_type.replace('-', '_')) try: config_fn = getattr(self, method_name) except AttributeError: raise ValueError( "A native worker of type '{}' is configured but " "there is no method '{}' on {}".format( worker_type, method_name, type(self))) yield config_fn(worker_logname, worker_id, worker) self.log.info( "Ok, {worker_logname} configured", worker_logname=worker_logname, ) d.addCallback(configure_worker, worker_logname, worker_type, worker_id, worker) # guest worker processes setup elif worker_type == u'guest': # now actually start the (guest) worker .. # FIXME: start_worker() takes the whole configuration item for guest workers, whereas native workers # only take the options (which is part of the whole config item for the worker) d = self._controller.call(u'crossbar.start_worker', worker_id, worker_type, worker, options=CallOptions()) else: raise Exception( 'logic error: unexpected worker_type="{}"'.format( worker_type)) if parallel_worker_start: dl.append(d) else: yield d yield gatherResults(dl) self.log.info( hl('Ok, local node configuration booted successfully!', color='green', bold=True))
def _configure_native_worker_proxy(self, worker_logname, worker_id, worker): yield self._configure_native_worker_common(worker_logname, worker_id, worker) # start transports on proxy for i, transport in enumerate(worker.get('transports', [])): if 'id' in transport: transport_id = transport['id'] else: transport_id = 'transport{:03d}'.format(i) transport['id'] = transport_id self.log.info( "Order {worker_logname} to start Transport {transport_id}", worker_logname=worker_logname, transport_id=hlid(transport_id), ) # XXX we're doing startup, and begining proxy workers -- # want to share the web-transport etc etc stuff between # these and otehr kinds of routers / transports yield self._controller.call('crossbar.worker.{}.start_proxy_transport'.format(worker_id), transport_id, transport, options=CallOptions()) if transport['type'] == 'web': paths = transport.get('paths', {}) elif transport['type'] in ('universal'): paths = transport.get('web', {}).get('paths', {}) else: paths = None # Web service paths if paths: for path in sorted(paths): if path != '/': webservice = paths[path] if 'id' in webservice: webservice_id = webservice['id'] else: webservice_id = 'webservice{:03d}'.format(self._webservice_no) webservice['id'] = webservice_id self._webservice_no += 1 self.log.info( "Order Transport {transport_id} to start Web Service {webservice_id}", transport_id=hlid(transport_id), webservice_id=hlid(webservice_id), path=hluserid(path), ) yield self._controller.call('crossbar.worker.{}.start_web_transport_service'.format(worker_id), transport_id, path, webservice, options=CallOptions()) self.log.info( "Ok, Transport {transport_id} has started Web Service {webservice_id}", transport_id=hlid(transport_id), webservice_id=hlid(webservice_id), path=hluserid(path), ) self.log.info( "Ok, {worker_logname} has started Transport {transport_id}", worker_logname=worker_logname, transport_id=hlid(transport_id), ) # set up backend connections on the proxy for i, connection_name in enumerate(worker.get('connections', {})): self.log.debug( "Starting connection {index}: {name}", index=i, name=connection_name, ) yield self._controller.call( 'crossbar.worker.{}.start_proxy_connection'.format(worker_id), connection_name, worker['connections'].get(connection_name, {}), ) # set up realms and roles on the proxy for i, realm_name in enumerate(worker.get('routes', {})): roles = worker['routes'][realm_name] for role_id, connections in roles.items(): if not isinstance(connections, list): connections = [connections] # used to be a single string, now a list of strings for connection_id in connections: self.log.debug( "Starting proxy realm route {realm}, {role} to {connection}", realm=realm_name, role=role_id, connection=connection_id, ) yield self._controller.call( 'crossbar.worker.{}.start_proxy_realm_route'.format(worker_id), realm_name, {role_id: connection_id}, )
def _configure_native_worker_router(self, worker_logname, worker_id, worker): yield self._configure_native_worker_common(worker_logname, worker_id, worker) # start realms on router for realm in worker.get('realms', []): # start realm if 'id' in realm: realm_id = realm['id'] else: realm_id = 'realm{:03d}'.format(self._realm_no) realm['id'] = realm_id self._realm_no += 1 self.log.info( "Order {worker_logname} to start Realm {realm_id}", worker_logname=worker_logname, realm_id=hlid(realm_id), ) yield self._controller.call('crossbar.worker.{}.start_router_realm'.format(worker_id), realm_id, realm, options=CallOptions()) self.log.info( "Ok, {worker_logname} has started Realm {realm_id}", worker_logname=worker_logname, realm_id=hlid(realm_id), ) # add roles to realm for role in realm.get('roles', []): if 'id' in role: role_id = role['id'] else: role_id = 'role{:03d}'.format(self._role_no) role['id'] = role_id self._role_no += 1 self.log.info( "Order Realm {realm_id} to start Role {role_id}", realm_id=hlid(realm_id), role_id=hlid(role_id), ) yield self._controller.call('crossbar.worker.{}.start_router_realm_role'.format(worker_id), realm_id, role_id, role, options=CallOptions()) self.log.info( "Ok, Realm {realm_id} has started Role {role_id}", realm_id=hlid(realm_id), role_id=hlid(role_id), ) # start components to run embedded in the router for component in worker.get('components', []): if 'id' in component: component_id = component['id'] else: component_id = 'component{:03d}'.format(self._component_no) component['id'] = component_id self._component_no += 1 yield self._controller.call('crossbar.worker.{}.start_router_component'.format(worker_id), component_id, component, options=CallOptions()) self.log.info( "{logname}: component '{component}' started", logname=worker_logname, component=component_id, ) # start transports on router for transport in worker.get('transports', []): if 'id' in transport: transport_id = transport['id'] else: transport_id = 'transport{:03d}'.format(self._transport_no) transport['id'] = transport_id self._transport_no += 1 add_paths_on_transport_create = False self.log.info( "Order {worker_logname} to start Transport {transport_id}", worker_logname=worker_logname, transport_id=hlid(transport_id), ) yield self._controller.call('crossbar.worker.{}.start_router_transport'.format(worker_id), transport_id, transport, create_paths=add_paths_on_transport_create, options=CallOptions()) self.log.info( "Ok, {worker_logname} has started Transport {transport_id}", worker_logname=worker_logname, transport_id=hlid(transport_id), ) if not add_paths_on_transport_create: if transport['type'] == 'web': paths = transport.get('paths', {}) elif transport['type'] in ('universal'): paths = transport.get('web', {}).get('paths', {}) else: paths = None # Web service paths if paths: for path in sorted(paths): if path != '/': webservice = paths[path] if 'id' in webservice: webservice_id = webservice['id'] else: webservice_id = 'webservice{:03d}'.format(self._webservice_no) webservice['id'] = webservice_id self._webservice_no += 1 self.log.info( "Order Transport {transport_id} to start Web Service {webservice_id}", transport_id=hlid(transport_id), webservice_id=hlid(webservice_id), path=hluserid(path), ) yield self._controller.call( 'crossbar.worker.{}.start_web_transport_service'.format(worker_id), transport_id, path, webservice, options=CallOptions()) self.log.info( "Ok, Transport {transport_id} has started Web Service {webservice_id}", transport_id=hlid(transport_id), webservice_id=hlid(webservice_id), path=hluserid(path), ) # start rlinks for realms dl = [] for realm in worker.get('realms', []): realm_id = realm['id'] for i, rlink in enumerate(realm.get('rlinks', [])): if 'id' in rlink: rlink_id = rlink['id'] else: rlink_id = 'rlink{:03d}'.format(i) rlink['id'] = rlink_id self.log.info( 'Starting router-to-router "{rlink_id}" on realm "{realm_id}" ..', realm_id=hlid(realm_id), rlink_id=hlid(rlink_id), ) d = self._controller.call('crossbar.worker.{}.start_router_realm_link'.format(worker_id), realm_id, rlink_id, rlink, options=CallOptions()) def done(_): self.log.info( 'Ok, router-to-router {rlink_id} started on realm "{realm_id}".', realm_id=hlid(realm_id), rlink_id=hlid(rlink_id), ) d.addCallback(done) dl.append(d) # FIXME: rlinks must be started without waiting for them to be established. otherwise the start of other stuff # is waiting for all rlinks to be up! d = gatherResults(dl)