async def _setup_heartmonitor(
ctx,
ping_url,
pong_url,
monitor_url,
log_level=logging.INFO,
curve_publickey=None,
curve_secretkey=None,
**heart_monitor_kwargs,
):
"""Set up heart monitor
For use in a background process,
via Process(target=start_heartmonitor)
"""
ping_socket = ctx.socket(zmq.PUB)
bind(
ping_socket,
ping_url,
curve_publickey=curve_publickey,
curve_secretkey=curve_secretkey,
)
ping_stream = ZMQStream(ping_socket)
pong_socket = ctx.socket(zmq.ROUTER)
set_hwm(pong_socket, 0)
bind(
pong_socket,
pong_url,
curve_publickey=curve_publickey,
curve_secretkey=curve_secretkey,
)
pong_stream = ZMQStream(pong_socket)
monitor_socket = ctx.socket(zmq.XPUB)
connect(
monitor_socket,
monitor_url,
curve_publickey=curve_publickey,
curve_secretkey=curve_secretkey,
curve_serverkey=curve_publickey,
)
monitor_stream = ZMQStream(monitor_socket)
# reinitialize logging after fork
from .app import IPController
app = IPController(log_level=log_level)
heart_monitor_kwargs['log'] = app.log
heart_monitor = HeartMonitor(
ping_stream=ping_stream,
pong_stream=pong_stream,
monitor_stream=monitor_stream,
**heart_monitor_kwargs,
)
heart_monitor.start()
def bind(self, socket, url):
"""Bind a socket"""
return util.bind(
socket,
url,
curve_secretkey=self.curve_secretkey
if self.enable_curve else None,
curve_publickey=self.curve_publickey
if self.enable_curve else None,
)
def start_heartmonitor(
ping_url,
pong_url,
monitor_url,
log_level=logging.INFO,
curve_publickey=None,
curve_secretkey=None,
**kwargs,
):
"""Start a heart monitor.
For use in a background process,
via Process(target=start_heartmonitor)
"""
loop = ioloop.IOLoop()
loop.make_current()
ctx = zmq.Context()
ping_socket = ctx.socket(zmq.PUB)
bind(
ping_socket,
ping_url,
curve_publickey=curve_publickey,
curve_secretkey=curve_secretkey,
)
ping_stream = ZMQStream(ping_socket)
pong_socket = ctx.socket(zmq.ROUTER)
set_hwm(pong_socket, 0)
bind(
pong_socket,
pong_url,
curve_publickey=curve_publickey,
curve_secretkey=curve_secretkey,
)
pong_stream = ZMQStream(pong_socket)
monitor_socket = ctx.socket(zmq.XPUB)
connect(
monitor_socket,
monitor_url,
curve_publickey=curve_publickey,
curve_secretkey=curve_secretkey,
curve_serverkey=curve_publickey,
)
monitor_stream = ZMQStream(monitor_socket)
# reinitialize logging after fork
from .app import IPController
app = IPController(log_level=log_level)
kwargs['log'] = app.log
heart_monitor = HeartMonitor(
ping_stream=ping_stream,
pong_stream=pong_stream,
monitor_stream=monitor_stream,
**kwargs,
)
heart_monitor.start()
try:
loop.start()
finally:
loop.close(all_fds=True)
ctx.destroy()
def launch_broadcast_scheduler(
in_addr,
out_addrs,
mon_addr,
not_addr,
reg_addr,
identity,
config=None,
loglevel=logging.DEBUG,
log_url=None,
is_leaf=False,
in_thread=False,
outgoing_ids=None,
curve_publickey=None,
curve_secretkey=None,
depth=0,
max_depth=0,
scheduler_class=BroadcastScheduler,
logname='broadcast',
):
config, ctx, loop, mons, nots, querys, log = get_common_scheduler_streams(
mon_addr,
not_addr,
reg_addr,
config,
logname,
log_url,
loglevel,
in_thread,
curve_serverkey=curve_publickey,
curve_publickey=curve_publickey,
curve_secretkey=curve_secretkey,
)
is_root = depth == 0
sub_scheduler_id = get_id_with_prefix(identity)
incoming_stream = ZMQStream(ctx.socket(zmq.ROUTER), loop)
util.set_hwm(incoming_stream, 0)
incoming_stream.setsockopt(zmq.IDENTITY, sub_scheduler_id)
if is_root:
util.bind(incoming_stream, in_addr, curve_secretkey=curve_secretkey)
else:
util.connect(
incoming_stream,
in_addr,
curve_serverkey=curve_publickey,
curve_publickey=curve_publickey,
curve_secretkey=curve_secretkey,
)
outgoing_streams = []
for out_addr in out_addrs:
out = ZMQStream(ctx.socket(zmq.ROUTER), loop)
util.set_hwm(out, 0)
out.setsockopt(zmq.IDENTITY, sub_scheduler_id)
util.bind(out, out_addr, curve_secretkey=curve_secretkey)
outgoing_streams.append(out)
scheduler_args = dict(
client_stream=incoming_stream,
mon_stream=mons,
notifier_stream=nots,
query_stream=querys,
loop=loop,
log=log,
config=config,
depth=depth,
max_depth=max_depth,
name=identity,
)
if is_leaf:
scheduler_args.update(engine_stream=outgoing_streams[0], is_leaf=True)
else:
scheduler_args.update(
connected_sub_scheduler_ids=[
get_id_with_prefix(identity) for identity in outgoing_ids
],
outgoing_streams=outgoing_streams,
)
scheduler = scheduler_class(**scheduler_args)
scheduler.start()
if not in_thread:
try:
loop.start()
except KeyboardInterrupt:
scheduler.log.critical("Interrupted, exiting...")
def launch_scheduler(
scheduler_class,
in_addr,
out_addr,
mon_addr,
not_addr,
reg_addr,
config=None,
logname='root',
log_url=None,
loglevel=logging.DEBUG,
identity=None,
in_thread=False,
curve_secretkey=None,
curve_publickey=None,
):
config, ctx, loop, mons, nots, querys, log = get_common_scheduler_streams(
mon_addr,
not_addr,
reg_addr,
config,
logname,
log_url,
loglevel,
in_thread,
curve_serverkey=curve_publickey,
curve_publickey=curve_publickey,
curve_secretkey=curve_secretkey,
)
util.set_hwm(mons, 0)
ins = ZMQStream(ctx.socket(zmq.ROUTER), loop)
util.set_hwm(ins, 0)
if identity:
ins.setsockopt(zmq.IDENTITY, identity + b'_in')
util.bind(ins, in_addr, curve_secretkey=curve_secretkey)
outs = ZMQStream(ctx.socket(zmq.ROUTER), loop)
util.set_hwm(outs, 0)
if identity:
outs.setsockopt(zmq.IDENTITY, identity + b'_out')
util.bind(outs, out_addr, curve_secretkey=curve_secretkey)
scheduler = scheduler_class(
client_stream=ins,
engine_stream=outs,
mon_stream=mons,
notifier_stream=nots,
query_stream=querys,
loop=loop,
log=log,
config=config,
)
scheduler.start()
if not in_thread:
try:
loop.start()
except KeyboardInterrupt:
scheduler.log.critical("Interrupted, exiting...")