def setUp(self):
# create the io_loop
self._io_loop = IOLoop.instance()
# and the context
self._ctx = zmq.Context(1)
self._settings = Settings()
self._settings.ZEROMQ_MASTER_PUSH = 'inproc://spyder-zmq-master-push'
self._settings.ZEROMQ_WORKER_PROC_FETCHER_PULL = \
self._settings.ZEROMQ_MASTER_PUSH
self._settings.ZEROMQ_MASTER_SUB = 'inproc://spyder-zmq-master-sub'
self._settings.ZEROMQ_WORKER_PROC_EXTRACTOR_PUB = \
self._settings.ZEROMQ_MASTER_SUB
self._settings.ZEROMQ_MGMT_MASTER = 'inproc://spyder-zmq-mgmt-master'
self._settings.ZEROMQ_MGMT_WORKER = 'inproc://spyder-zmq-mgmt-worker'
# setup the mgmt sockets
self._setup_mgmt_sockets()
# setup the data sockets
self._setup_data_servers()
# setup the management interface
self._mgmt = ZmqMgmt(self._mgmt_sockets['worker_sub'],
self._mgmt_sockets['worker_pub'],
io_loop=self._io_loop)
self._mgmt.start()
self._mgmt.add_callback(ZMQ_SPYDER_MGMT_WORKER, self.on_mgmt_end)
def setUp(self):
# create the io_loop
self._io_loop = IOLoop.instance()
# and the context
self._ctx = zmq.Context(1)
self._settings = Settings()
self._settings.ZEROMQ_MASTER_PUSH = 'inproc://spyder-zmq-master-push'
self._settings.ZEROMQ_WORKER_PROC_FETCHER_PULL = \
self._settings.ZEROMQ_MASTER_PUSH
self._settings.ZEROMQ_MASTER_SUB = 'inproc://spyder-zmq-master-sub'
self._settings.ZEROMQ_WORKER_PROC_EXTRACTOR_PUB = \
self._settings.ZEROMQ_MASTER_SUB
self._settings.ZEROMQ_MGMT_MASTER = 'inproc://spyder-zmq-mgmt-master'
self._settings.ZEROMQ_MGMT_WORKER = 'inproc://spyder-zmq-mgmt-worker'
# setup the mgmt sockets
self._setup_mgmt_sockets()
# setup the data sockets
self._setup_data_servers()
# setup the management interface
self._mgmt = ZmqMgmt( self._mgmt_sockets['worker_sub'],
self._mgmt_sockets['worker_pub'], io_loop=self._io_loop)
self._mgmt.start()
self._mgmt.add_callback(ZMQ_SPYDER_MGMT_WORKER, self.on_mgmt_end)
def setUp(self):
# create the io_loop
self._io_loop = IOLoop.instance()
# and the context
self._ctx = zmq.Context(1)
# setup the mgmt sockets
self._setup_mgmt_sockets()
# setup the data sockets
self._setup_data_sockets()
# setup the management interface
self._mgmt = ZmqMgmt(self._mgmt_sockets['worker_sub'],
self._mgmt_sockets['worker_pub'],
io_loop=self._io_loop)
self._mgmt.start()
self._mgmt.add_callback(ZMQ_SPYDER_MGMT_WORKER, self.on_mgmt_end)
def create_worker_management(settings, zmq_context, io_loop):
"""
Create and return a new instance of the `ZmqMgmt`.
"""
listening_socket = zmq_context.socket(zmq.SUB)
listening_socket.setsockopt(zmq.SUBSCRIBE, "")
listening_socket.connect(settings.ZEROMQ_MGMT_MASTER)
publishing_socket = zmq_context.socket(zmq.PUB)
publishing_socket.connect(settings.ZEROMQ_MGMT_WORKER)
return ZmqMgmt(listening_socket, publishing_socket, io_loop=io_loop)
def create_master_management(settings, zmq_context, io_loop):
"""
Create the management interface for master processes.
"""
listening_socket = zmq_context.socket(zmq.SUB)
listening_socket.setsockopt(zmq.SUBSCRIBE, "")
listening_socket.bind(settings.ZEROMQ_MGMT_WORKER)
publishing_socket = zmq_context.socket(zmq.PUB)
publishing_socket.bind(settings.ZEROMQ_MGMT_MASTER)
return ZmqMgmt(listening_socket, publishing_socket, io_loop=io_loop)
def setUp(self):
# create the io_loop
self._io_loop = IOLoop.instance()
# and the context
self._ctx = zmq.Context(1)
# setup the mgmt sockets
self._setup_mgmt_sockets()
# setup the data sockets
self._setup_data_sockets()
# setup the management interface
self._mgmt = ZmqMgmt(self._mgmt_sockets["worker_sub"], self._mgmt_sockets["worker_pub"], io_loop=self._io_loop)
self._mgmt.start()
self._mgmt.add_callback(ZMQ_SPYDER_MGMT_WORKER, self.on_mgmt_end)
def test_simple_mgmt_session(self):
mgmt = ZmqMgmt(self._worker_sub, self._worker_pub, io_loop=self._io_loop)
mgmt.start()
self.assertRaises(ValueError, mgmt.add_callback, "test", "test")
mgmt.add_callback(self._topic, self.call_me)
mgmt.add_callback(ZMQ_SPYDER_MGMT_WORKER, self.on_end)
test_msg = MgmtMessage(topic=self._topic, data='test'.encode())
self._master_pub.send_multipart(test_msg.serialize())
def assert_correct_mgmt_answer(raw_msg):
msg = MgmtMessage(raw_msg)
self.assertEqual(ZMQ_SPYDER_MGMT_WORKER_QUIT_ACK, msg.data)
mgmt.remove_callback(self._topic, self.call_me)
mgmt.remove_callback(ZMQ_SPYDER_MGMT_WORKER, self.on_end)
self.assertEqual({}, mgmt._callbacks)
self._master_sub.on_recv(assert_correct_mgmt_answer)
self._io_loop.start()
class ZmqTornadoIntegrationTest(unittest.TestCase):
def setUp(self):
# create the io_loop
self._io_loop = IOLoop.instance()
# and the context
self._ctx = zmq.Context(1)
# setup the mgmt sockets
self._setup_mgmt_sockets()
# setup the data sockets
self._setup_data_sockets()
# setup the management interface
self._mgmt = ZmqMgmt(self._mgmt_sockets['worker_sub'],
self._mgmt_sockets['worker_pub'],
io_loop=self._io_loop)
self._mgmt.start()
self._mgmt.add_callback(ZMQ_SPYDER_MGMT_WORKER, self.on_mgmt_end)
def tearDown(self):
# stop the mgmt
self._mgmt.stop()
# close all sockets
for socket in self._mgmt_sockets.itervalues():
socket.close()
for socket in self._worker_sockets.itervalues():
socket.close()
# terminate the context
self._ctx.term()
def _setup_mgmt_sockets(self):
self._mgmt_sockets = dict()
# adress for the communication from master to worker(s)
mgmt_master_worker = 'inproc://master/worker/coordination/'
# connect the master with the worker
# the master is a ZMQStream because we are sending msgs from the test
sock = self._ctx.socket(zmq.PUB)
sock.bind(mgmt_master_worker)
self._mgmt_sockets['tmp1'] = sock
self._mgmt_sockets['master_pub'] = ZMQStream(sock, self._io_loop)
# the worker stream is created inside the ZmqMgmt class
self._mgmt_sockets['worker_sub'] = self._ctx.socket(zmq.SUB)
self._mgmt_sockets['worker_sub'].setsockopt(zmq.SUBSCRIBE, "")
self._mgmt_sockets['worker_sub'].connect(mgmt_master_worker)
# adress for the communication from worker(s) to master
mgmt_worker_master = 'inproc://worker/master/coordination/'
# connect the worker with the master
self._mgmt_sockets['worker_pub'] = self._ctx.socket(zmq.PUB)
self._mgmt_sockets['worker_pub'].bind(mgmt_worker_master)
sock = self._ctx.socket(zmq.SUB)
sock.setsockopt(zmq.SUBSCRIBE, "")
sock.connect(mgmt_worker_master)
self._mgmt_sockets['tmp2'] = sock
self._mgmt_sockets['master_sub'] = ZMQStream(sock, self._io_loop)
def _setup_data_sockets(self):
self._worker_sockets = dict()
# address for master -> worker communication
data_master_worker = 'inproc://master/worker/pipeline/'
sock = self._ctx.socket(zmq.PUSH)
sock.bind(data_master_worker)
self._worker_sockets['tmp3'] = sock
self._worker_sockets['master_push'] = ZMQStream(sock, self._io_loop)
self._worker_sockets['worker_pull'] = self._ctx.socket(zmq.PULL)
self._worker_sockets['worker_pull'].connect(data_master_worker)
# address for worker -> master communication
data_worker_master = 'inproc://worker/master/pipeline/'
self._worker_sockets['worker_pub'] = self._ctx.socket(zmq.PUB)
self._worker_sockets['worker_pub'].bind(data_worker_master)
sock = self._ctx.socket(zmq.SUB)
sock.setsockopt(zmq.SUBSCRIBE, "")
sock.connect(data_worker_master)
self._worker_sockets['tmp4'] = sock
self._worker_sockets['master_sub'] = ZMQStream(sock, self._io_loop)
def on_mgmt_end(self, _msg):
self._io_loop.stop()
class ZmqTornadoIntegrationTest(unittest.TestCase):
def setUp(self):
# create the io_loop
self._io_loop = IOLoop.instance()
# and the context
self._ctx = zmq.Context(1)
# setup the mgmt sockets
self._setup_mgmt_sockets()
# setup the data sockets
self._setup_data_sockets()
# setup the management interface
self._mgmt = ZmqMgmt(self._mgmt_sockets["worker_sub"], self._mgmt_sockets["worker_pub"], io_loop=self._io_loop)
self._mgmt.start()
self._mgmt.add_callback(ZMQ_SPYDER_MGMT_WORKER, self.on_mgmt_end)
def tearDown(self):
# stop the mgmt
self._mgmt.stop()
# close all sockets
for socket in self._mgmt_sockets.itervalues():
socket.close()
for socket in self._worker_sockets.itervalues():
socket.close()
# terminate the context
self._ctx.term()
def _setup_mgmt_sockets(self):
self._mgmt_sockets = dict()
# adress for the communication from master to worker(s)
mgmt_master_worker = "inproc://master/worker/coordination/"
# connect the master with the worker
# the master is a ZMQStream because we are sending msgs from the test
sock = self._ctx.socket(zmq.PUB)
sock.bind(mgmt_master_worker)
self._mgmt_sockets["tmp1"] = sock
self._mgmt_sockets["master_pub"] = ZMQStream(sock, self._io_loop)
# the worker stream is created inside the ZmqMgmt class
self._mgmt_sockets["worker_sub"] = self._ctx.socket(zmq.SUB)
self._mgmt_sockets["worker_sub"].setsockopt(zmq.SUBSCRIBE, "")
self._mgmt_sockets["worker_sub"].connect(mgmt_master_worker)
# adress for the communication from worker(s) to master
mgmt_worker_master = "inproc://worker/master/coordination/"
# connect the worker with the master
self._mgmt_sockets["worker_pub"] = self._ctx.socket(zmq.PUB)
self._mgmt_sockets["worker_pub"].bind(mgmt_worker_master)
sock = self._ctx.socket(zmq.SUB)
sock.setsockopt(zmq.SUBSCRIBE, "")
sock.connect(mgmt_worker_master)
self._mgmt_sockets["tmp2"] = sock
self._mgmt_sockets["master_sub"] = ZMQStream(sock, self._io_loop)
def _setup_data_sockets(self):
self._worker_sockets = dict()
# address for master -> worker communication
data_master_worker = "inproc://master/worker/pipeline/"
sock = self._ctx.socket(zmq.PUSH)
sock.bind(data_master_worker)
self._worker_sockets["tmp3"] = sock
self._worker_sockets["master_push"] = ZMQStream(sock, self._io_loop)
self._worker_sockets["worker_pull"] = self._ctx.socket(zmq.PULL)
self._worker_sockets["worker_pull"].connect(data_master_worker)
# address for worker -> master communication
data_worker_master = "inproc://worker/master/pipeline/"
self._worker_sockets["worker_pub"] = self._ctx.socket(zmq.PUB)
self._worker_sockets["worker_pub"].bind(data_worker_master)
sock = self._ctx.socket(zmq.SUB)
sock.setsockopt(zmq.SUBSCRIBE, "")
sock.connect(data_worker_master)
self._worker_sockets["tmp4"] = sock
self._worker_sockets["master_sub"] = ZMQStream(sock, self._io_loop)
def on_mgmt_end(self, _msg):
self._io_loop.stop()
class ZmqTornadoIntegrationTest(unittest.TestCase):
def setUp(self):
# create the io_loop
self._io_loop = IOLoop.instance()
# and the context
self._ctx = zmq.Context(1)
self._settings = Settings()
self._settings.ZEROMQ_MASTER_PUSH = 'inproc://spyder-zmq-master-push'
self._settings.ZEROMQ_WORKER_PROC_FETCHER_PULL = \
self._settings.ZEROMQ_MASTER_PUSH
self._settings.ZEROMQ_MASTER_SUB = 'inproc://spyder-zmq-master-sub'
self._settings.ZEROMQ_WORKER_PROC_EXTRACTOR_PUB = \
self._settings.ZEROMQ_MASTER_SUB
self._settings.ZEROMQ_MGMT_MASTER = 'inproc://spyder-zmq-mgmt-master'
self._settings.ZEROMQ_MGMT_WORKER = 'inproc://spyder-zmq-mgmt-worker'
# setup the mgmt sockets
self._setup_mgmt_sockets()
# setup the data sockets
self._setup_data_servers()
# setup the management interface
self._mgmt = ZmqMgmt( self._mgmt_sockets['worker_sub'],
self._mgmt_sockets['worker_pub'], io_loop=self._io_loop)
self._mgmt.start()
self._mgmt.add_callback(ZMQ_SPYDER_MGMT_WORKER, self.on_mgmt_end)
def tearDown(self):
# stop the mgmt
self._mgmt.stop()
# close all sockets
for socket in self._mgmt_sockets.itervalues():
socket.close()
for socket in self._worker_sockets.itervalues():
socket.close()
# terminate the context
self._ctx.term()
def _setup_mgmt_sockets(self):
self._mgmt_sockets = dict()
# adress for the communication from master to worker(s)
mgmt_master_worker = self._settings.ZEROMQ_MGMT_MASTER
# connect the master with the worker
# the master is a ZMQStream because we are sending msgs from the test
sock = self._ctx.socket(zmq.PUB)
sock.bind(mgmt_master_worker)
self._mgmt_sockets['tmp1'] = sock
self._mgmt_sockets['master_pub'] = ZMQStream(sock, self._io_loop)
# the worker stream is created inside the ZmqMgmt class
self._mgmt_sockets['worker_sub'] = self._ctx.socket(zmq.SUB)
self._mgmt_sockets['worker_sub'].setsockopt(zmq.SUBSCRIBE, "")
self._mgmt_sockets['worker_sub'].connect(mgmt_master_worker)
# adress for the communication from worker(s) to master
mgmt_worker_master = self._settings.ZEROMQ_MGMT_WORKER
# connect the worker with the master
self._mgmt_sockets['worker_pub'] = self._ctx.socket(zmq.PUB)
self._mgmt_sockets['worker_pub'].bind(mgmt_worker_master)
sock = self._ctx.socket(zmq.SUB)
sock.setsockopt(zmq.SUBSCRIBE, "")
sock.connect(mgmt_worker_master)
self._mgmt_sockets['tmp2'] = sock
self._mgmt_sockets['master_sub'] = ZMQStream(sock, self._io_loop)
def _setup_data_servers(self):
self._worker_sockets = dict()
# address for master -> worker communication
data_master_worker = self._settings.ZEROMQ_WORKER_PROC_EXTRACTOR_PULL
sock = self._ctx.socket(zmq.PUSH)
sock.bind(data_master_worker)
self._worker_sockets['tmp3'] = sock
self._worker_sockets['master_push'] = ZMQStream(sock, self._io_loop)
# address for worker -> master communication
data_worker_master = self._settings.ZEROMQ_WORKER_PROC_EXTRACTOR_PUB
sock = self._ctx.socket(zmq.SUB)
sock.setsockopt(zmq.SUBSCRIBE, "")
sock.bind(data_worker_master)
self._worker_sockets['tmp4'] = sock
self._worker_sockets['master_sub'] = ZMQStream(sock, self._io_loop)
def on_mgmt_end(self, _msg):
self._io_loop.stop()