async def _setup_heartmonitor(
ctx,
ping_url,
pong_url,
monitor_url,
log_level=logging.INFO,
curve_publickey=None,
curve_secretkey=None,
**heart_monitor_kwargs,
):
"""Set up heart monitor
For use in a background process,
via Process(target=start_heartmonitor)
"""
ping_socket = ctx.socket(zmq.PUB)
bind(
ping_socket,
ping_url,
curve_publickey=curve_publickey,
curve_secretkey=curve_secretkey,
)
ping_stream = ZMQStream(ping_socket)
pong_socket = ctx.socket(zmq.ROUTER)
set_hwm(pong_socket, 0)
bind(
pong_socket,
pong_url,
curve_publickey=curve_publickey,
curve_secretkey=curve_secretkey,
)
pong_stream = ZMQStream(pong_socket)
monitor_socket = ctx.socket(zmq.XPUB)
connect(
monitor_socket,
monitor_url,
curve_publickey=curve_publickey,
curve_secretkey=curve_secretkey,
curve_serverkey=curve_publickey,
)
monitor_stream = ZMQStream(monitor_socket)
# reinitialize logging after fork
from .app import IPController
app = IPController(log_level=log_level)
heart_monitor_kwargs['log'] = app.log
heart_monitor = HeartMonitor(
ping_stream=ping_stream,
pong_stream=pong_stream,
monitor_stream=monitor_stream,
**heart_monitor_kwargs,
)
heart_monitor.start()
def start(self):
self.log.info(
f"Starting kernel nanny for engine {self.engine_id}, pid={self.pid}, nanny pid={os.getpid()}"
)
self._watcher_thread = Thread(target=self.wait_for_parent_thread,
name="WatchParent",
daemon=True)
self._watcher_thread.start()
# ignore SIGINT sent to parent
signal.signal(signal.SIGINT, signal.SIG_IGN)
self.loop = IOLoop.current()
self.context = zmq.Context()
# set up control socket (connection to Scheduler)
self.control_socket = self.context.socket(zmq.ROUTER)
self.control_socket.identity = self.identity
util.connect(
self.control_socket,
self.control_url,
curve_serverkey=self.curve_serverkey,
)
self.control_stream = ZMQStream(self.control_socket)
self.control_stream.on_recv_stream(self.dispatch_control)
# set up relay socket (connection to parent's control socket)
self.parent_socket = self.context.socket(zmq.DEALER)
if self.curve_secretkey:
self.parent_socket.setsockopt(zmq.CURVE_SERVER, 1)
self.parent_socket.setsockopt(zmq.CURVE_SECRETKEY,
self.curve_secretkey)
port = self.parent_socket.bind_to_random_port("tcp://127.0.0.1")
# now that we've bound, pass port to parent via AsyncResult
self.pipe.write(f"tcp://127.0.0.1:{port}\n")
if not sys.platform.startswith("win"):
# watch for the stdout pipe to close
# as a signal that our parent is shutting down
self.loop.add_handler(self.pipe, self.pipe_handler,
IOLoop.READ | IOLoop.ERROR)
self.parent_stream = ZMQStream(self.parent_socket)
self.parent_stream.on_recv_stream(self.dispatch_parent)
try:
self.loop.start()
finally:
self.loop.close(all_fds=True)
self.context.term()
try:
self.pipe.close()
except BrokenPipeError:
pass
self.log.debug("exiting")
def notify_exit(self):
"""Notify the Hub that our parent has exited"""
self.log.info("Notifying Hub that our parent has shut down")
s = self.context.socket(zmq.DEALER)
# finite, nonzero LINGER to prevent hang without dropping message during exit
s.LINGER = 3000
util.connect(
s,
self.registration_url,
curve_serverkey=self.curve_serverkey,
curve_secretkey=self.curve_secretkey,
curve_publickey=self.curve_publickey,
)
self.session.send(s,
"unregistration_request",
content={"id": self.engine_id})
s.close()
def connect(s, addr):
return util.connect(
s,
addr,
curve_serverkey=curve_serverkey,
curve_secretkey=curve_secretkey,
curve_publickey=curve_publickey,
)
def connect(self, socket, url):
return util.connect(
socket,
url,
curve_serverkey=self.curve_publickey
if self.enable_curve else None,
curve_publickey=self.curve_publickey
if self.enable_curve else None,
curve_secretkey=self.curve_secretkey
if self.enable_curve else None,
)
def start_heartmonitor(
ping_url,
pong_url,
monitor_url,
log_level=logging.INFO,
curve_publickey=None,
curve_secretkey=None,
**kwargs,
):
"""Start a heart monitor.
For use in a background process,
via Process(target=start_heartmonitor)
"""
loop = ioloop.IOLoop()
loop.make_current()
ctx = zmq.Context()
ping_socket = ctx.socket(zmq.PUB)
bind(
ping_socket,
ping_url,
curve_publickey=curve_publickey,
curve_secretkey=curve_secretkey,
)
ping_stream = ZMQStream(ping_socket)
pong_socket = ctx.socket(zmq.ROUTER)
set_hwm(pong_socket, 0)
bind(
pong_socket,
pong_url,
curve_publickey=curve_publickey,
curve_secretkey=curve_secretkey,
)
pong_stream = ZMQStream(pong_socket)
monitor_socket = ctx.socket(zmq.XPUB)
connect(
monitor_socket,
monitor_url,
curve_publickey=curve_publickey,
curve_secretkey=curve_secretkey,
curve_serverkey=curve_publickey,
)
monitor_stream = ZMQStream(monitor_socket)
# reinitialize logging after fork
from .app import IPController
app = IPController(log_level=log_level)
kwargs['log'] = app.log
heart_monitor = HeartMonitor(
ping_stream=ping_stream,
pong_stream=pong_stream,
monitor_stream=monitor_stream,
**kwargs,
)
heart_monitor.start()
try:
loop.start()
finally:
loop.close(all_fds=True)
ctx.destroy()
def launch_broadcast_scheduler(
in_addr,
out_addrs,
mon_addr,
not_addr,
reg_addr,
identity,
config=None,
loglevel=logging.DEBUG,
log_url=None,
is_leaf=False,
in_thread=False,
outgoing_ids=None,
curve_publickey=None,
curve_secretkey=None,
depth=0,
max_depth=0,
scheduler_class=BroadcastScheduler,
logname='broadcast',
):
config, ctx, loop, mons, nots, querys, log = get_common_scheduler_streams(
mon_addr,
not_addr,
reg_addr,
config,
logname,
log_url,
loglevel,
in_thread,
curve_serverkey=curve_publickey,
curve_publickey=curve_publickey,
curve_secretkey=curve_secretkey,
)
is_root = depth == 0
sub_scheduler_id = get_id_with_prefix(identity)
incoming_stream = ZMQStream(ctx.socket(zmq.ROUTER), loop)
util.set_hwm(incoming_stream, 0)
incoming_stream.setsockopt(zmq.IDENTITY, sub_scheduler_id)
if is_root:
util.bind(incoming_stream, in_addr, curve_secretkey=curve_secretkey)
else:
util.connect(
incoming_stream,
in_addr,
curve_serverkey=curve_publickey,
curve_publickey=curve_publickey,
curve_secretkey=curve_secretkey,
)
outgoing_streams = []
for out_addr in out_addrs:
out = ZMQStream(ctx.socket(zmq.ROUTER), loop)
util.set_hwm(out, 0)
out.setsockopt(zmq.IDENTITY, sub_scheduler_id)
util.bind(out, out_addr, curve_secretkey=curve_secretkey)
outgoing_streams.append(out)
scheduler_args = dict(
client_stream=incoming_stream,
mon_stream=mons,
notifier_stream=nots,
query_stream=querys,
loop=loop,
log=log,
config=config,
depth=depth,
max_depth=max_depth,
name=identity,
)
if is_leaf:
scheduler_args.update(engine_stream=outgoing_streams[0], is_leaf=True)
else:
scheduler_args.update(
connected_sub_scheduler_ids=[
get_id_with_prefix(identity) for identity in outgoing_ids
],
outgoing_streams=outgoing_streams,
)
scheduler = scheduler_class(**scheduler_args)
scheduler.start()
if not in_thread:
try:
loop.start()
except KeyboardInterrupt:
scheduler.log.critical("Interrupted, exiting...")