def _streamer(host, task_in_port, task_out_port):
# streamer for zmq workers
try:
z.zmq.proxy(z.bind('tcp://%s:%s' % (host, task_in_port), z.zmq.PULL),
z.bind('tcp://%s:%s' % (host, task_out_port), z.zmq.PUSH))
except (KeyboardInterrupt, z.zmq.ZMQError):
pass # killed cleanly by SIGINT/SIGTERM
def _streamer():
# streamer for zmq workers running on the master node
port = int(config.zworkers.ctrl_port)
task_input_url = 'tcp://127.0.0.1:%d' % (port + 2)
task_output_url = 'tcp://%s:%s' % (config.dbserver.listen, port + 1)
try:
z.zmq.proxy(z.bind(task_input_url, z.zmq.PULL),
z.bind(task_output_url, z.zmq.PUSH))
except (KeyboardInterrupt, z.zmq.ContextTerminated):
pass # killed cleanly by SIGINT/SIGTERM
def _streamer(host):
# streamer for zmq workers
port = int(config.zworkers.ctrl_port)
task_input_url = 'tcp://127.0.0.1:%d' % (port + 2)
task_server_url = 'tcp://%s:%s' % (host, port + 1)
try:
z.zmq.proxy(z.bind(task_input_url, z.zmq.PULL),
z.bind(task_server_url, z.zmq.PUSH))
except (KeyboardInterrupt, z.zmq.ContextTerminated):
pass # killed cleanly by SIGINT/SIGTERM
def streamer(host, task_in_port, task_out_port):
"""
A streamer for zmq workers.
:param host: name or IP of the controller node
:param task_in_port: port where to send the tasks
:param task_out_port: port from where to receive the tasks
"""
try:
z.zmq.proxy(z.bind('tcp://%s:%s' % (host, task_in_port), z.zmq.PULL),
z.bind('tcp://%s:%s' % (host, task_out_port), z.zmq.PUSH))
except (KeyboardInterrupt, z.zmq.ZMQError):
pass # killed cleanly by SIGINT/SIGTERM
def start(self):
"""
Start database worker threads
"""
# give a nice name to the process
w.setproctitle('oq-dbserver')
dworkers = []
for _ in range(self.num_workers):
sock = z.Socket(self.backend, z.zmq.REP, 'connect')
threading.Thread(target=self.dworker, args=(sock,)).start()
dworkers.append(sock)
logging.warning('DB server started with %s on %s, pid %d',
sys.executable, self.frontend, self.pid)
if ZMQ:
# start task_in->task_out streamer thread
c = config.zworkers
threading.Thread(
target=w._streamer,
args=(self.master_host, c.task_in_port, c.task_out_port)
).start()
logging.warning('Task streamer started from %s -> %s',
c.task_in_port, c.task_out_port)
# start zworkers and wait a bit for them
msg = self.master.start()
logging.warning(msg)
time.sleep(1)
# start frontend->backend proxy for the database workers
try:
z.zmq.proxy(z.bind(self.frontend, z.zmq.ROUTER),
z.bind(self.backend, z.zmq.DEALER))
except (KeyboardInterrupt, z.zmq.ZMQError):
for sock in dworkers:
sock.running = False
sock.zsocket.close()
logging.warning('DB server stopped')
finally:
self.stop()
def start(self):
"""
Start database worker threads
"""
# give a nice name to the process
w.setproctitle('oq-dbserver')
dworkers = []
for _ in range(self.num_workers):
sock = z.Socket(self.backend, z.zmq.REP, 'connect')
threading.Thread(target=self.dworker, args=(sock,)).start()
dworkers.append(sock)
logging.warning('DB server started with %s on %s, pid %d',
sys.executable, self.frontend, self.pid)
if ZMQ:
# start task_in->task_out streamer thread
c = config.zworkers
threading.Thread(
target=w._streamer,
args=(self.master_host, c.task_in_port, c.task_out_port)
).start()
logging.warning('Task streamer started from %s -> %s',
c.task_in_port, c.task_out_port)
# start zworkers and wait a bit for them
msg = self.master.start()
logging.warning(msg)
time.sleep(1)
# start frontend->backend proxy for the database workers
try:
z.zmq.proxy(z.bind(self.frontend, z.zmq.ROUTER),
z.bind(self.backend, z.zmq.DEALER))
except (KeyboardInterrupt, z.zmq.ZMQError):
for sock in dworkers:
sock.running = False
sock.zsocket.close()
logging.warning('DB server stopped')
finally:
self.stop()