def start(self): """Starts a new child worker process""" args = [control.get_process_command(), '--worker', '-f', '-s', '-P'] if self.threadpoolsize: args.append('--threadpoolsize=%d' % self.threadpoolsize) endpoint = ProcessEndpoint(reactor, control.get_process_command(), args, os.environ) factory = protocol.Factory() factory.protocol = lambda: ProcessAMP(is_worker=False, locator=JobHandler()) self.process = yield endpoint.connect(factory) self.process.lost_handler = self._worker_died self.started_at = datetime.datetime.now() self._logger.debug("Started new worker %r", self) if ipdevpoll_conf.getboolean("multiprocess", "ping_workers", fallback=True): self._ping_loop.start( interval=ipdevpoll_conf.getint("multiprocess", "ping_interval", fallback=30), now=False, ) returnValue(self)
def start(): from twisted.internet import reactor ep = ProcessEndpoint(reactor, RUST_PATH, [RUST_PATH], childFDs={ 0: 'w', 1: 'r', 2: 2 }, env=None) factory = Factory.forProtocol(Responder) rust = yield ep.connect(factory) call1 = rust.callRemote(Sum, a=1, b=2) call2 = rust.callRemote(Sum, a=4, b=-100) dl = yield DeferredList([call1, call2]) print(dl) d = Deferred() reactor.callLater(.01, d.callback, None) # Add small delay because it is hard to check when the request fd # has been closed. yield d reactor.stop()
def start(self): args = [control.get_process_command(), '--worker', '-f', '-s', '-P'] if self.threadpoolsize: args.append('--threadpoolsize=%d' % self.threadpoolsize) endpoint = ProcessEndpoint(reactor, control.get_process_command(), args, os.environ) factory = protocol.Factory() factory.protocol = lambda: ProcessAMP(is_worker=False, locator=JobHandler()) self.process = yield endpoint.connect(factory) self.process.lost_handler = self._worker_died returnValue(self)
def start(self): """Starts a new child worker process""" args = [control.get_process_command(), '--worker', '-f', '-s', '-P'] if self.threadpoolsize: args.append('--threadpoolsize=%d' % self.threadpoolsize) endpoint = ProcessEndpoint(reactor, control.get_process_command(), args, os.environ) factory = protocol.Factory() factory.protocol = lambda: ProcessAMP(is_worker=False, locator=JobHandler()) self.process = yield endpoint.connect(factory) self.process.lost_handler = self._worker_died self.started_at = datetime.datetime.now() self._logger.debug("Started new worker %r", self) returnValue(self)
def start_process(self, config): """ Management API for starting a new process on this node. :param config: The process configuration. :type config: dict :returns: int -- The PID of the new process. """ if config['type'] in ['router', 'component.python']: ## ## start a Crossbar.io worker process ## filename = pkg_resources.resource_filename('crossbar', 'worker.py') args = [executable, "-u", filename] if self.debug: args.append('--debug') if sys.platform == 'win32': args.extend(['--logfile', 'test.log']) ep = ProcessEndpoint(self._node._reactor, executable, args, errFlag = StandardErrorBehavior.DROP, env = os.environ) else: ep = ProcessEndpoint(self._node._reactor, executable, args, childFDs = {0: 'w', 1: 'r', 2: 2}, # does not work on Windows errFlag = StandardErrorBehavior.LOG, env = os.environ) ready = Deferred() d = ep.connect(self._node._router_client_transport_factory) def onconnect(res): pid = res.transport.pid self._processes[pid] = NodeControllerSession.NodeProcess('worker', pid, ready) def onerror(err): ready.errback(err) d.addCallbacks(onconnect, onerror) return ready elif config['type'] == 'component.program': ## ## start a program process ## from twisted.internet import protocol from twisted.internet import reactor from twisted.internet.error import ProcessDone, ProcessTerminated import re, json class ProgramWorkerProcess(protocol.ProcessProtocol): def __init__(self): self._pid = None def connectionMade(self): if 'stdin' in config and config['stdin'] == 'config' and 'config' in config: ## write process config from configuration to stdin ## of the forked process and close stdin self.transport.write(json.dumps(config['config'])) self.transport.closeStdin() def outReceived(self, data): if 'stdout' in config and config['stdout'] == 'log': try: data = str(data).strip() except: data = "{} bytes".format(len(data)) log.msg("Worker {} (stdout): {}".format(self._pid, data)) def errReceived(self, data): if 'stderr' in config and config['stderr'] == 'log': try: data = str(data).strip() except: data = "{} bytes".format(len(data)) log.msg("Worker {} (stderr): {}".format(self._pid, data)) def inConnectionLost(self): pass def outConnectionLost(self): pass def errConnectionLost(self): pass def processExited(self, reason): pass def processEnded(self, reason): if isinstance(reason.value, ProcessDone): log.msg("Worker {}: Ended cleanly.".format(self._pid)) elif isinstance(reason.value, ProcessTerminated): log.msg("Worker {}: Ended with error {}".format(self._pid, reason.value.exitCode)) else: ## should not arrive here pass exe = config['executable'] args = [exe] args.extend(config.get('arguments', [])) workdir = self._node._cbdir if 'workdir' in config: workdir = os.path.join(workdir, config['workdir']) workdir = os.path.abspath(workdir) ready = Deferred() proto = ProgramWorkerProcess() try: trnsp = reactor.spawnProcess(proto, exe, args, path = workdir, env = os.environ) except Exception as e: log.msg("Worker: Program could not be started - {}".format(e)) ready.errback(e) else: pid = trnsp.pid proto._pid = pid self._processes[pid] = NodeControllerSession.NodeProcess('program', pid, ready) log.msg("Worker {}: Program started.".format(pid)) ready.callback(pid) return ready else: raise ApplicationError("wamp.error.invalid_argument", "Invalid process type '{}'".format(config['type']))
executable, args, #childFDs = {0: 'w', 1: 'r', 2: 2}, # does not work on Windows errFlag = StandardErrorBehavior.LOG, env = os.environ) else: from crossbar.process import CustomProcessEndpoint ep = CustomProcessEndpoint(reactor, executable, args, name = "Worker", env = os.environ) d = ep.connect(transport_factory) def onconnect(res): pid = res.transport.pid log.msg("Worker forked with PID {}".format(pid)) def onerror(err): log.msg("Could not fork worker: {}".format(err.value)) d.addCallbacks(onconnect, onerror) ## now enter the Twisted reactor loop ## reactor.run()