def launch_scheduler(in_addr, out_addr, mon_addr, not_addr, reg_addr, config=None,
logname='root', log_url=None, loglevel=logging.DEBUG,
identity=b'task', in_thread=False):
ZMQStream = zmqstream.ZMQStream
if config:
# unwrap dict back into Config
config = Config(config)
if in_thread:
# use instance() to get the same Context/Loop as our parent
ctx = zmq.Context.instance()
loop = ioloop.IOLoop.current()
else:
# in a process, don't use instance()
# for safety with multiprocessing
ctx = zmq.Context()
loop = ioloop.IOLoop()
ins = ZMQStream(ctx.socket(zmq.ROUTER),loop)
util.set_hwm(ins, 0)
ins.setsockopt(zmq.IDENTITY, identity + b'_in')
ins.bind(in_addr)
outs = ZMQStream(ctx.socket(zmq.ROUTER),loop)
util.set_hwm(outs, 0)
outs.setsockopt(zmq.IDENTITY, identity + b'_out')
outs.bind(out_addr)
mons = zmqstream.ZMQStream(ctx.socket(zmq.PUB),loop)
util.set_hwm(mons, 0)
mons.connect(mon_addr)
nots = zmqstream.ZMQStream(ctx.socket(zmq.SUB),loop)
nots.setsockopt(zmq.SUBSCRIBE, b'')
nots.connect(not_addr)
querys = ZMQStream(ctx.socket(zmq.DEALER),loop)
querys.connect(reg_addr)
# setup logging.
if in_thread:
log = Application.instance().log
else:
if log_url:
log = connect_logger(logname, ctx, log_url, root="scheduler", loglevel=loglevel)
else:
log = local_logger(logname, loglevel)
scheduler = TaskScheduler(client_stream=ins, engine_stream=outs,
mon_stream=mons, notifier_stream=nots,
query_stream=querys,
loop=loop, log=log,
config=config)
scheduler.start()
if not in_thread:
try:
loop.start()
except KeyboardInterrupt:
scheduler.log.critical("Interrupted, exiting...")
def get_common_scheduler_streams(
mon_addr,
not_addr,
reg_addr,
config,
logname,
log_url,
loglevel,
in_thread,
curve_serverkey,
curve_publickey,
curve_secretkey,
):
if config:
# unwrap dict back into Config
config = Config(config)
if in_thread:
# use instance() to get the same Context/Loop as our parent
ctx = zmq.Context.instance()
loop = ioloop.IOLoop.current()
else:
# in a process, don't use instance()
# for safety with multiprocessing
ctx = zmq.Context()
loop = ioloop.IOLoop(make_current=False)
def connect(s, addr):
return util.connect(
s,
addr,
curve_serverkey=curve_serverkey,
curve_secretkey=curve_secretkey,
curve_publickey=curve_publickey,
)
mons = zmqstream.ZMQStream(ctx.socket(zmq.PUB), loop)
connect(mons, mon_addr)
nots = zmqstream.ZMQStream(ctx.socket(zmq.SUB), loop)
nots.setsockopt(zmq.SUBSCRIBE, b'')
connect(nots, not_addr)
querys = ZMQStream(ctx.socket(zmq.DEALER), loop)
connect(querys, reg_addr)
# setup logging.
if in_thread:
log = traitlets.log.get_logger()
else:
if log_url:
log = connect_logger(logname,
ctx,
log_url,
root="scheduler",
loglevel=loglevel)
else:
log = local_logger(logname, loglevel)
return config, ctx, loop, mons, nots, querys, log
def launch_scheduler(in_addr, out_addr, mon_addr, not_addr, reg_addr, config=None,
logname='root', log_url=None, loglevel=logging.DEBUG,
identity=b'task', in_thread=False):
ZMQStream = zmqstream.ZMQStream
if config:
# unwrap dict back into Config
config = Config(config)
if in_thread:
# use instance() to get the same Context/Loop as our parent
ctx = zmq.Context.instance()
loop = ioloop.IOLoop.instance()
else:
# in a process, don't use instance()
# for safety with multiprocessing
ctx = zmq.Context()
loop = ioloop.IOLoop()
ins = ZMQStream(ctx.socket(zmq.ROUTER),loop)
util.set_hwm(ins, 0)
ins.setsockopt(zmq.IDENTITY, identity + b'_in')
ins.bind(in_addr)
outs = ZMQStream(ctx.socket(zmq.ROUTER),loop)
util.set_hwm(outs, 0)
outs.setsockopt(zmq.IDENTITY, identity + b'_out')
outs.bind(out_addr)
mons = zmqstream.ZMQStream(ctx.socket(zmq.PUB),loop)
util.set_hwm(mons, 0)
mons.connect(mon_addr)
nots = zmqstream.ZMQStream(ctx.socket(zmq.SUB),loop)
nots.setsockopt(zmq.SUBSCRIBE, b'')
nots.connect(not_addr)
querys = ZMQStream(ctx.socket(zmq.DEALER),loop)
querys.connect(reg_addr)
# setup logging.
if in_thread:
log = Application.instance().log
else:
if log_url:
log = connect_logger(logname, ctx, log_url, root="scheduler", loglevel=loglevel)
else:
log = local_logger(logname, loglevel)
scheduler = TaskScheduler(client_stream=ins, engine_stream=outs,
mon_stream=mons, notifier_stream=nots,
query_stream=querys,
loop=loop, log=log,
config=config)
scheduler.start()
if not in_thread:
try:
loop.start()
except KeyboardInterrupt:
scheduler.log.critical("Interrupted, exiting...")
def get_common_scheduler_streams(mon_addr, not_addr, reg_addr, config, logname,
log_url, loglevel, in_thread):
if config:
# unwrap dict back into Config
config = Config(config)
if in_thread:
# use instance() to get the same Context/Loop as our parent
ctx = zmq.Context.instance()
loop = ioloop.IOLoop.current()
else:
# in a process, don't use instance()
# for safety with multiprocessing
ctx = zmq.Context()
loop = ioloop.IOLoop()
mons = zmqstream.ZMQStream(ctx.socket(zmq.PUB), loop)
mons.connect(mon_addr)
nots = zmqstream.ZMQStream(ctx.socket(zmq.SUB), loop)
nots.setsockopt(zmq.SUBSCRIBE, b'')
nots.connect(not_addr)
querys = ZMQStream(ctx.socket(zmq.DEALER), loop)
querys.connect(reg_addr)
# setup logging.
if in_thread:
log = Application.instance().log
else:
if log_url:
log = connect_logger(logname,
ctx,
log_url,
root="scheduler",
loglevel=loglevel)
else:
log = local_logger(logname, loglevel)
return config, ctx, loop, mons, nots, querys, log
def __init__(
self,
*,
pid: int,
engine_id: int,
control_url: str,
registration_url: str,
identity: bytes,
curve_serverkey: bytes,
curve_publickey: bytes,
curve_secretkey: bytes,
config: Config,
pipe,
log_level: int = logging.INFO,
):
self.pid = pid
self.engine_id = engine_id
self.parent_process = psutil.Process(self.pid)
self.control_url = control_url
self.registration_url = registration_url
self.identity = identity
self.curve_serverkey = curve_serverkey
self.curve_publickey = curve_publickey
self.curve_secretkey = curve_secretkey
self.config = config
self.pipe = pipe
self.session = Session(config=self.config)
self.log = local_logger(f"{self.__class__.__name__}.{engine_id}",
log_level)
self.log.propagate = False
self.control_handlers = {
"signal_request": self.signal_request,
}
self._finish_called = False
