def _streamer(host, task_in_port, task_out_port): # streamer for zmq workers try: z.zmq.proxy(z.bind('tcp://%s:%s' % (host, task_in_port), z.zmq.PULL), z.bind('tcp://%s:%s' % (host, task_out_port), z.zmq.PUSH)) except (KeyboardInterrupt, z.zmq.ZMQError): pass # killed cleanly by SIGINT/SIGTERM
def _streamer(): # streamer for zmq workers running on the master node port = int(config.zworkers.ctrl_port) task_input_url = 'tcp://127.0.0.1:%d' % (port + 2) task_output_url = 'tcp://%s:%s' % (config.dbserver.listen, port + 1) try: z.zmq.proxy(z.bind(task_input_url, z.zmq.PULL), z.bind(task_output_url, z.zmq.PUSH)) except (KeyboardInterrupt, z.zmq.ContextTerminated): pass # killed cleanly by SIGINT/SIGTERM
def _streamer(host): # streamer for zmq workers port = int(config.zworkers.ctrl_port) task_input_url = 'tcp://127.0.0.1:%d' % (port + 2) task_server_url = 'tcp://%s:%s' % (host, port + 1) try: z.zmq.proxy(z.bind(task_input_url, z.zmq.PULL), z.bind(task_server_url, z.zmq.PUSH)) except (KeyboardInterrupt, z.zmq.ContextTerminated): pass # killed cleanly by SIGINT/SIGTERM
def streamer(host, task_in_port, task_out_port): """ A streamer for zmq workers. :param host: name or IP of the controller node :param task_in_port: port where to send the tasks :param task_out_port: port from where to receive the tasks """ try: z.zmq.proxy(z.bind('tcp://%s:%s' % (host, task_in_port), z.zmq.PULL), z.bind('tcp://%s:%s' % (host, task_out_port), z.zmq.PUSH)) except (KeyboardInterrupt, z.zmq.ZMQError): pass # killed cleanly by SIGINT/SIGTERM
def start(self): """ Start database worker threads """ # give a nice name to the process w.setproctitle('oq-dbserver') dworkers = [] for _ in range(self.num_workers): sock = z.Socket(self.backend, z.zmq.REP, 'connect') threading.Thread(target=self.dworker, args=(sock,)).start() dworkers.append(sock) logging.warning('DB server started with %s on %s, pid %d', sys.executable, self.frontend, self.pid) if ZMQ: # start task_in->task_out streamer thread c = config.zworkers threading.Thread( target=w._streamer, args=(self.master_host, c.task_in_port, c.task_out_port) ).start() logging.warning('Task streamer started from %s -> %s', c.task_in_port, c.task_out_port) # start zworkers and wait a bit for them msg = self.master.start() logging.warning(msg) time.sleep(1) # start frontend->backend proxy for the database workers try: z.zmq.proxy(z.bind(self.frontend, z.zmq.ROUTER), z.bind(self.backend, z.zmq.DEALER)) except (KeyboardInterrupt, z.zmq.ZMQError): for sock in dworkers: sock.running = False sock.zsocket.close() logging.warning('DB server stopped') finally: self.stop()
def start(self): """ Start database worker threads """ # give a nice name to the process w.setproctitle('oq-dbserver') dworkers = [] for _ in range(self.num_workers): sock = z.Socket(self.backend, z.zmq.REP, 'connect') threading.Thread(target=self.dworker, args=(sock,)).start() dworkers.append(sock) logging.warning('DB server started with %s on %s, pid %d', sys.executable, self.frontend, self.pid) if ZMQ: # start task_in->task_out streamer thread c = config.zworkers threading.Thread( target=w._streamer, args=(self.master_host, c.task_in_port, c.task_out_port) ).start() logging.warning('Task streamer started from %s -> %s', c.task_in_port, c.task_out_port) # start zworkers and wait a bit for them msg = self.master.start() logging.warning(msg) time.sleep(1) # start frontend->backend proxy for the database workers try: z.zmq.proxy(z.bind(self.frontend, z.zmq.ROUTER), z.bind(self.backend, z.zmq.DEALER)) except (KeyboardInterrupt, z.zmq.ZMQError): for sock in dworkers: sock.running = False sock.zsocket.close() logging.warning('DB server stopped') finally: self.stop()