class ManagerControlled(object):
def __init__(self, *args, **kwargs):
self.context = Context.instance()
self.loop = IOLoop.instance()
self.control_socket = self.context.socket(SUB)
self.control_socket.setsockopt(LINGER, 0) # discard unsent messages on close
self.control_socket.setsockopt(SUBSCRIBE, '')
self.control_socket.connect('tcp://{}:{}'.format(MANAGER_PUB_ADDRESS, MANAGER_PUB_PORT))
self.control_stream = ZMQStream(self.control_socket, self.loop)
self.control_stream.on_recv_stream(self.control_handler)
def control_handler(self, stream, message_list):
for message in message_list:
try:
notification, data = message.split()
except ValueError:
notification = message
if notification == NOTIFICATION_PROCESS_STOP:
self.stop()
def stop(self):
self.control_stream.stop_on_recv()
self.control_stream.close()
self.control_socket.close()
class Controller:
def __init__(self, cfg, target=None, args=[]):
(self.target, self.args) = (target, args)
# Set up server socket
self.context = zmq.Context()
self.socket = self.context.socket(zmq.ROUTER)
self.port = self.socket.bind_to_random_port('tcp://*',
min_port=cfg["port_min"],
max_port=cfg["port_max"],
max_tries=100)
self.uri = "tcp://127.0.0.1:%d" % (self.port)
print "Controller listening on %s" % (self.uri)
self.start()
def send_json(self, identity, obj):
self.stream.send(identity, zmq.SNDMORE)
self.stream.send_json(obj)
def start(self):
# Launch worker subprocesses
self.processes = dict(
zip(self.args, map(lambda x: self.target(x, self.uri), self.args)))
for d, p in self.processes.iteritems():
print "Launching Worker: ", d
p.start()
# Start IOLoop
self.stream = ZMQStream(self.socket)
self.ioloop = ioloop.IOLoop.instance()
self.ioloop.call_later(1.0, self.on_start)
self.stream.on_recv_stream(self.on_rcv)
try:
self.ioloop.start()
except KeyboardInterrupt:
print " ** Got keyboard interrupt (^C) shutting down cleanly ... ** "
self.shutdown()
self.ioloop.close()
for d, p in self.processes.iteritems():
p.join()
def on_start(self):
print "Controller: start doing stuff"
def on_rcv(self, stream, msg):
[identity, msg] = msg
#print identity,msg
msg = json.loads(msg)
op = msg[0]
if (hasattr(self, "on_" + op)):
f = getattr(self, "on_" + op)
args = [identity] + msg[1:]
f(*args)
else:
print "Controller received invalid command: ", msg
def shutdown(self):
#self.socket.send_json( ("shutdown",) )
self.ioloop.stop()
def main():
import sys
from PyQt4 import QtGui, Qt
from eventloop import QtLoop, ZMQLoop
from kernel import Kernel
from epubsub.hub import Hub
import zmq
from zmq.eventloop.zmqstream import ZMQStream
def callback(stream, msg):
print msg
stream.send('OK')
print 'running'
#app = QtGui.QApplication(sys.argv)
loop = ZMQLoop()
loop.install()
kernel = Kernel()
ctx = zmq.Context()
socket = ctx.socket(zmq.ROUTER)
socket.bind('tcp://127.0.0.1:11111')
stream = ZMQStream(socket)
stream.on_recv_stream(callback)
loop.start()
def rec(port):
zmq_ctx = zmq.Context()
s = zmq_ctx.socket(zmq.SUB)
s.bind('tcp://*:{port}'.format(port=port))
s.setsockopt(zmq.SUBSCRIBE, b"")
stream = ZMQStream(s)
stream.on_recv_stream(rec_frame)
ioloop.IOLoop.instance().start()
while True:
pass
def bind(self, interface):
context = zmq.Context()
socket = context.socket(zmq.REP)
socket.bind(interface)
stream = ZMQStream(socket)
stream.on_recv_stream(self.dispatch)
class KernelNanny:
"""Object for monitoring
Must be child of engine
Handles signal messages and watches Engine process for exiting
"""
def __init__(
self,
*,
pid: int,
engine_id: int,
control_url: str,
registration_url: str,
identity: bytes,
curve_serverkey: bytes,
curve_publickey: bytes,
curve_secretkey: bytes,
config: Config,
pipe,
log_level: int = logging.INFO,
):
self.pid = pid
self.engine_id = engine_id
self.parent_process = psutil.Process(self.pid)
self.control_url = control_url
self.registration_url = registration_url
self.identity = identity
self.curve_serverkey = curve_serverkey
self.curve_publickey = curve_publickey
self.curve_secretkey = curve_secretkey
self.config = config
self.pipe = pipe
self.session = Session(config=self.config)
self.log = local_logger(f"{self.__class__.__name__}.{engine_id}",
log_level)
self.log.propagate = False
self.control_handlers = {
"signal_request": self.signal_request,
}
self._finish_called = False
def wait_for_parent_thread(self):
"""Wait for my parent to exit, then I'll notify the controller and shut down"""
self.log.info(f"Nanny watching parent pid {self.pid}.")
while True:
try:
exit_code = self.parent_process.wait(60)
except psutil.TimeoutExpired:
continue
else:
break
self.log.critical(f"Parent {self.pid} exited with status {exit_code}.")
self.loop.add_callback(self.finish)
def pipe_handler(self, fd, events):
self.log.debug(f"Pipe event {events}")
self.loop.remove_handler(fd)
try:
fd.close()
except BrokenPipeError:
pass
try:
status = self.parent_process.wait(0)
except psutil.TimeoutExpired:
try:
status = self.parent_process.status()
except psutil.NoSuchProcess:
status = "exited"
self.log.critical(
f"Pipe closed, parent {self.pid} has status: {status}")
self.finish()
def notify_exit(self):
"""Notify the Hub that our parent has exited"""
self.log.info("Notifying Hub that our parent has shut down")
s = self.context.socket(zmq.DEALER)
# finite, nonzero LINGER to prevent hang without dropping message during exit
s.LINGER = 3000
util.connect(
s,
self.registration_url,
curve_serverkey=self.curve_serverkey,
curve_secretkey=self.curve_secretkey,
curve_publickey=self.curve_publickey,
)
self.session.send(s,
"unregistration_request",
content={"id": self.engine_id})
s.close()
def finish(self):
"""Prepare to exit and stop our event loop."""
if self._finish_called:
return
self._finish_called = True
self.notify_exit()
self.loop.add_callback(self.loop.stop)
def dispatch_control(self, stream, raw_msg):
"""Dispatch message from the control scheduler
If we have a handler registered"""
try:
idents, msg_frames = self.session.feed_identities(raw_msg)
except Exception as e:
self.log.error(f"Bad control message: {raw_msg}", exc_info=True)
return
try:
msg = self.session.deserialize(msg_frames, content=True)
except Exception:
content = error.wrap_exception()
self.log.error("Bad control message: %r",
msg_frames,
exc_info=True)
return
msg_type = msg['header']['msg_type']
if msg_type.endswith("_request"):
reply_type = msg_type[-len("_request"):]
else:
reply_type = "error"
self.log.debug(f"Client {idents[-1]} requested {msg_type}")
handler = self.control_handlers.get(msg_type, None)
if handler is None:
# don't have an intercept handler, relay original message to parent
self.log.debug(f"Relaying {msg_type} {msg['header']['msg_id']}")
self.parent_stream.send_multipart(raw_msg)
return
try:
content = handler(msg['content'])
except Exception:
content = error.wrap_exception()
self.log.error("Error handling request: %r",
msg_type,
exc_info=True)
self.session.send(stream,
reply_type,
ident=idents,
content=content,
parent=msg)
def dispatch_parent(self, stream, raw_msg):
"""Relay messages from parent directly to control stream"""
self.control_stream.send_multipart(raw_msg)
# intercept message handlers
def signal_request(self, content):
"""Handle a signal request: send signal to parent process"""
sig = content['sig']
if isinstance(sig, str):
sig = getattr(signal, sig)
self.log.info(f"Sending signal {sig} to pid {self.pid}")
# exception will be caught and wrapped by the caller
self.parent_process.send_signal(sig)
return {"status": "ok"}
def start(self):
self.log.info(
f"Starting kernel nanny for engine {self.engine_id}, pid={self.pid}, nanny pid={os.getpid()}"
)
self._watcher_thread = Thread(target=self.wait_for_parent_thread,
name="WatchParent",
daemon=True)
self._watcher_thread.start()
# ignore SIGINT sent to parent
signal.signal(signal.SIGINT, signal.SIG_IGN)
self.loop = IOLoop.current()
self.context = zmq.Context()
# set up control socket (connection to Scheduler)
self.control_socket = self.context.socket(zmq.ROUTER)
self.control_socket.identity = self.identity
util.connect(
self.control_socket,
self.control_url,
curve_serverkey=self.curve_serverkey,
)
self.control_stream = ZMQStream(self.control_socket)
self.control_stream.on_recv_stream(self.dispatch_control)
# set up relay socket (connection to parent's control socket)
self.parent_socket = self.context.socket(zmq.DEALER)
if self.curve_secretkey:
self.parent_socket.setsockopt(zmq.CURVE_SERVER, 1)
self.parent_socket.setsockopt(zmq.CURVE_SECRETKEY,
self.curve_secretkey)
port = self.parent_socket.bind_to_random_port("tcp://127.0.0.1")
# now that we've bound, pass port to parent via AsyncResult
self.pipe.write(f"tcp://127.0.0.1:{port}\n")
if not sys.platform.startswith("win"):
# watch for the stdout pipe to close
# as a signal that our parent is shutting down
self.loop.add_handler(self.pipe, self.pipe_handler,
IOLoop.READ | IOLoop.ERROR)
self.parent_stream = ZMQStream(self.parent_socket)
self.parent_stream.on_recv_stream(self.dispatch_parent)
try:
self.loop.start()
finally:
self.loop.close(all_fds=True)
self.context.term()
try:
self.pipe.close()
except BrokenPipeError:
pass
self.log.debug("exiting")
@classmethod
def main(cls, *args, **kwargs):
"""Main body function.
Instantiates and starts a nanny.
Args and keyword args passed to the constructor.
Should be called in a subprocess.
"""
# start a new event loop for the forked process
asyncio.set_event_loop(asyncio.new_event_loop())
IOLoop().make_current()
self = cls(*args, **kwargs)
self.start()
def bind(self, port):
context = zmq.Context()
socket = context.socket(zmq.REP)
socket.bind(port)
stream = ZMQStream(socket)
stream.on_recv_stream(self.dispatch)
context = zmq.Context()
socket = context.socket(zmq.REQ)
socket.connect("tcp://127.0.0.1:5556")
stream = ZMQStream(socket)
def echo(stream, msg):
global this
global last
this = int(time.time())
global tc
global total
tc += 1
total += 1
if this != last:
print str(tc) + ' ' + str(total)
tc = 0
last = this
# stream.send_multipart(msg)
stream.on_recv_stream(echo)
def send():
stream.send(b"Hello")
ioloop.IOLoop.instance().add_callback(send)
ioloop.IOLoop.instance().add_callback(send)
ioloop.IOLoop.instance().start()
if sys.argv[1] == 'server':
# Setup a REP ZMQ socket
context = zmq.Context()
socket = context.socket(zmq.REP)
socket.bind('tcp://127.0.0.1:8889')
# See http://zeromq.github.io/pyzmq/eventloop.html
# Setup the event loop
# loop = tornado.ioloop.IOLoop.instance()
ioloop.install()
# Add a ZMQStream made from the socket to the event loop
stream = ZMQStream(socket)
# Register recv callback
stream.on_recv_stream(recv_callback_stream)
# How to add a periodic callback to the event loop
periodic = ioloop.PeriodicCallback(periodic_callback, 1000)
# periodic_callback will be called each second
periodic.start()
# Setup tornado webserver on :8888
app.listen(8888)
# And start the event loop
print 'Open a browser to http://localhost:8888'
ioloop.IOLoop.instance().start()
elif sys.argv[1] == 'client':
if len(sys.argv) != 5:
if sys.argv[1] == 'server':
# Setup a REP ZMQ socket
context = zmq.Context()
socket = context.socket(zmq.REP)
socket.bind('tcp://127.0.0.1:8889')
# See http://zeromq.github.io/pyzmq/eventloop.html
# Setup the event loop
# loop = tornado.ioloop.IOLoop.instance()
ioloop.install()
# Add a ZMQStream made from the socket to the event loop
stream = ZMQStream(socket)
# Register recv callback
stream.on_recv_stream(recv_callback_stream)
# How to add a periodic callback to the event loop
periodic = ioloop.PeriodicCallback(periodic_callback, 1000)
# periodic_callback will be called each second
periodic.start()
# Setup tornado webserver on :8888
app.listen(8888)
# And start the event loop
print 'Open a browser to http://localhost:8888'
ioloop.IOLoop.instance().start()
elif sys.argv[1] == 'client':
if len(sys.argv) != 5:
socket.send_multipart(msg)
time.sleep(0.1)
run_event = threading.Event()
run_event.set()
t = threading.Thread(target=threaded_loop)
t.daemon = False
t.start()
# listeners can request the knownStreams dictionary on a req-rep socket
info_socket = context.socket(zmq.REP)
info_socket.bind("tcp://*:5557")
info_stream = ZMQStream(info_socket)
info_stream.on_recv_stream(processInfoMsg)
try:
ioloop.IOLoop.instance().start()
except KeyboardInterrupt:
print("Halt signal detected. Cleaning up...")
except:
print "Exception in server code:"
print '-'*60
traceback.print_exc(file=sys.stdout)
print '-'*60
print("Cleaning up...")
finally:
run_event.clear()
t.join()
info_socket.close()
time.sleep(0.1)
run_event = threading.Event()
run_event.set()
t = threading.Thread(target=threaded_loop)
t.daemon = False
t.start()
# listeners can request the knownStreams dictionary on a req-rep socket
info_socket = context.socket(zmq.REP)
info_socket.bind("tcp://*:5557")
info_stream = ZMQStream(info_socket)
info_stream.on_recv_stream(processInfoMsg)
try:
ioloop.IOLoop.instance().start()
except KeyboardInterrupt:
print("Halt signal detected. Cleaning up...")
except:
print "Exception in server code:"
print '-' * 60
traceback.print_exc(file=sys.stdout)
print '-' * 60
print("Cleaning up...")
finally:
run_event.clear()
t.join()
info_socket.close()
