Example #1
0
    def setUp(self):

        # create the io_loop
        self._io_loop = IOLoop.instance()

        # and the context
        self._ctx = zmq.Context(1)

        self._settings = Settings()
        self._settings.ZEROMQ_MASTER_PUSH = 'inproc://spyder-zmq-master-push'
        self._settings.ZEROMQ_WORKER_PROC_FETCHER_PULL = \
            self._settings.ZEROMQ_MASTER_PUSH
        self._settings.ZEROMQ_MASTER_SUB = 'inproc://spyder-zmq-master-sub'
        self._settings.ZEROMQ_WORKER_PROC_EXTRACTOR_PUB = \
            self._settings.ZEROMQ_MASTER_SUB

        self._settings.ZEROMQ_MGMT_MASTER = 'inproc://spyder-zmq-mgmt-master'
        self._settings.ZEROMQ_MGMT_WORKER = 'inproc://spyder-zmq-mgmt-worker'

        # setup the mgmt sockets
        self._setup_mgmt_sockets()

        # setup the data sockets
        self._setup_data_servers()

        # setup the management interface
        self._mgmt = ZmqMgmt(self._mgmt_sockets['worker_sub'],
                             self._mgmt_sockets['worker_pub'],
                             io_loop=self._io_loop)
        self._mgmt.start()
        self._mgmt.add_callback(ZMQ_SPYDER_MGMT_WORKER, self.on_mgmt_end)
    def setUp(self):

        # create the io_loop
        self._io_loop = IOLoop.instance()

        # and the context
        self._ctx = zmq.Context(1)

        self._settings = Settings()
        self._settings.ZEROMQ_MASTER_PUSH = 'inproc://spyder-zmq-master-push'
        self._settings.ZEROMQ_WORKER_PROC_FETCHER_PULL = \
            self._settings.ZEROMQ_MASTER_PUSH
        self._settings.ZEROMQ_MASTER_SUB = 'inproc://spyder-zmq-master-sub'
        self._settings.ZEROMQ_WORKER_PROC_EXTRACTOR_PUB = \
            self._settings.ZEROMQ_MASTER_SUB

        self._settings.ZEROMQ_MGMT_MASTER = 'inproc://spyder-zmq-mgmt-master'
        self._settings.ZEROMQ_MGMT_WORKER = 'inproc://spyder-zmq-mgmt-worker'

        # setup the mgmt sockets
        self._setup_mgmt_sockets()

        # setup the data sockets
        self._setup_data_servers()

        # setup the management interface
        self._mgmt = ZmqMgmt( self._mgmt_sockets['worker_sub'],
            self._mgmt_sockets['worker_pub'], io_loop=self._io_loop)
        self._mgmt.start()
        self._mgmt.add_callback(ZMQ_SPYDER_MGMT_WORKER, self.on_mgmt_end)
    def setUp(self):

        # create the io_loop
        self._io_loop = IOLoop.instance()

        # and the context
        self._ctx = zmq.Context(1)

        # setup the mgmt sockets
        self._setup_mgmt_sockets()

        # setup the data sockets
        self._setup_data_sockets()

        # setup the management interface
        self._mgmt = ZmqMgmt(self._mgmt_sockets['worker_sub'],
                             self._mgmt_sockets['worker_pub'],
                             io_loop=self._io_loop)
        self._mgmt.start()
        self._mgmt.add_callback(ZMQ_SPYDER_MGMT_WORKER, self.on_mgmt_end)
Example #4
0
def create_worker_management(settings, zmq_context, io_loop):
    """
    Create and return a new instance of the `ZmqMgmt`.
    """
    listening_socket = zmq_context.socket(zmq.SUB)
    listening_socket.setsockopt(zmq.SUBSCRIBE, "")
    listening_socket.connect(settings.ZEROMQ_MGMT_MASTER)

    publishing_socket = zmq_context.socket(zmq.PUB)
    publishing_socket.connect(settings.ZEROMQ_MGMT_WORKER)

    return ZmqMgmt(listening_socket, publishing_socket, io_loop=io_loop)
Example #5
0
def create_master_management(settings, zmq_context, io_loop):
    """
    Create the management interface for master processes.
    """
    listening_socket = zmq_context.socket(zmq.SUB)
    listening_socket.setsockopt(zmq.SUBSCRIBE, "")
    listening_socket.bind(settings.ZEROMQ_MGMT_WORKER)

    publishing_socket = zmq_context.socket(zmq.PUB)
    publishing_socket.bind(settings.ZEROMQ_MGMT_MASTER)

    return ZmqMgmt(listening_socket, publishing_socket, io_loop=io_loop)
Example #6
0
    def setUp(self):

        # create the io_loop
        self._io_loop = IOLoop.instance()

        # and the context
        self._ctx = zmq.Context(1)

        # setup the mgmt sockets
        self._setup_mgmt_sockets()

        # setup the data sockets
        self._setup_data_sockets()

        # setup the management interface
        self._mgmt = ZmqMgmt(self._mgmt_sockets["worker_sub"], self._mgmt_sockets["worker_pub"], io_loop=self._io_loop)
        self._mgmt.start()
        self._mgmt.add_callback(ZMQ_SPYDER_MGMT_WORKER, self.on_mgmt_end)
Example #7
0
    def test_simple_mgmt_session(self):
        
        mgmt = ZmqMgmt(self._worker_sub, self._worker_pub, io_loop=self._io_loop)
        mgmt.start()

        self.assertRaises(ValueError, mgmt.add_callback, "test", "test")

        mgmt.add_callback(self._topic, self.call_me)
        mgmt.add_callback(ZMQ_SPYDER_MGMT_WORKER, self.on_end)

        test_msg = MgmtMessage(topic=self._topic, data='test'.encode())
        self._master_pub.send_multipart(test_msg.serialize())

        def assert_correct_mgmt_answer(raw_msg):
            msg = MgmtMessage(raw_msg)
            self.assertEqual(ZMQ_SPYDER_MGMT_WORKER_QUIT_ACK, msg.data)
            mgmt.remove_callback(self._topic, self.call_me)
            mgmt.remove_callback(ZMQ_SPYDER_MGMT_WORKER, self.on_end)
            self.assertEqual({}, mgmt._callbacks)

        self._master_sub.on_recv(assert_correct_mgmt_answer)

        self._io_loop.start()
class ZmqTornadoIntegrationTest(unittest.TestCase):
    def setUp(self):

        # create the io_loop
        self._io_loop = IOLoop.instance()

        # and the context
        self._ctx = zmq.Context(1)

        # setup the mgmt sockets
        self._setup_mgmt_sockets()

        # setup the data sockets
        self._setup_data_sockets()

        # setup the management interface
        self._mgmt = ZmqMgmt(self._mgmt_sockets['worker_sub'],
                             self._mgmt_sockets['worker_pub'],
                             io_loop=self._io_loop)
        self._mgmt.start()
        self._mgmt.add_callback(ZMQ_SPYDER_MGMT_WORKER, self.on_mgmt_end)

    def tearDown(self):
        # stop the mgmt
        self._mgmt.stop()

        # close all sockets
        for socket in self._mgmt_sockets.itervalues():
            socket.close()
        for socket in self._worker_sockets.itervalues():
            socket.close()

        # terminate the context
        self._ctx.term()

    def _setup_mgmt_sockets(self):

        self._mgmt_sockets = dict()

        # adress for the communication from master to worker(s)
        mgmt_master_worker = 'inproc://master/worker/coordination/'

        # connect the master with the worker
        # the master is a ZMQStream because we are sending msgs from the test
        sock = self._ctx.socket(zmq.PUB)
        sock.bind(mgmt_master_worker)
        self._mgmt_sockets['tmp1'] = sock
        self._mgmt_sockets['master_pub'] = ZMQStream(sock, self._io_loop)
        # the worker stream is created inside the ZmqMgmt class
        self._mgmt_sockets['worker_sub'] = self._ctx.socket(zmq.SUB)
        self._mgmt_sockets['worker_sub'].setsockopt(zmq.SUBSCRIBE, "")
        self._mgmt_sockets['worker_sub'].connect(mgmt_master_worker)

        # adress for the communication from worker(s) to master
        mgmt_worker_master = 'inproc://worker/master/coordination/'

        # connect the worker with the master
        self._mgmt_sockets['worker_pub'] = self._ctx.socket(zmq.PUB)
        self._mgmt_sockets['worker_pub'].bind(mgmt_worker_master)
        sock = self._ctx.socket(zmq.SUB)
        sock.setsockopt(zmq.SUBSCRIBE, "")
        sock.connect(mgmt_worker_master)
        self._mgmt_sockets['tmp2'] = sock
        self._mgmt_sockets['master_sub'] = ZMQStream(sock, self._io_loop)

    def _setup_data_sockets(self):

        self._worker_sockets = dict()

        # address for master -> worker communication
        data_master_worker = 'inproc://master/worker/pipeline/'

        sock = self._ctx.socket(zmq.PUSH)
        sock.bind(data_master_worker)
        self._worker_sockets['tmp3'] = sock
        self._worker_sockets['master_push'] = ZMQStream(sock, self._io_loop)
        self._worker_sockets['worker_pull'] = self._ctx.socket(zmq.PULL)
        self._worker_sockets['worker_pull'].connect(data_master_worker)

        # address for worker -> master communication
        data_worker_master = 'inproc://worker/master/pipeline/'

        self._worker_sockets['worker_pub'] = self._ctx.socket(zmq.PUB)
        self._worker_sockets['worker_pub'].bind(data_worker_master)
        sock = self._ctx.socket(zmq.SUB)
        sock.setsockopt(zmq.SUBSCRIBE, "")
        sock.connect(data_worker_master)
        self._worker_sockets['tmp4'] = sock
        self._worker_sockets['master_sub'] = ZMQStream(sock, self._io_loop)

    def on_mgmt_end(self, _msg):
        self._io_loop.stop()
Example #9
0
class ZmqTornadoIntegrationTest(unittest.TestCase):
    def setUp(self):

        # create the io_loop
        self._io_loop = IOLoop.instance()

        # and the context
        self._ctx = zmq.Context(1)

        # setup the mgmt sockets
        self._setup_mgmt_sockets()

        # setup the data sockets
        self._setup_data_sockets()

        # setup the management interface
        self._mgmt = ZmqMgmt(self._mgmt_sockets["worker_sub"], self._mgmt_sockets["worker_pub"], io_loop=self._io_loop)
        self._mgmt.start()
        self._mgmt.add_callback(ZMQ_SPYDER_MGMT_WORKER, self.on_mgmt_end)

    def tearDown(self):
        # stop the mgmt
        self._mgmt.stop()

        # close all sockets
        for socket in self._mgmt_sockets.itervalues():
            socket.close()
        for socket in self._worker_sockets.itervalues():
            socket.close()

        # terminate the context
        self._ctx.term()

    def _setup_mgmt_sockets(self):

        self._mgmt_sockets = dict()

        # adress for the communication from master to worker(s)
        mgmt_master_worker = "inproc://master/worker/coordination/"

        # connect the master with the worker
        # the master is a ZMQStream because we are sending msgs from the test
        sock = self._ctx.socket(zmq.PUB)
        sock.bind(mgmt_master_worker)
        self._mgmt_sockets["tmp1"] = sock
        self._mgmt_sockets["master_pub"] = ZMQStream(sock, self._io_loop)
        # the worker stream is created inside the ZmqMgmt class
        self._mgmt_sockets["worker_sub"] = self._ctx.socket(zmq.SUB)
        self._mgmt_sockets["worker_sub"].setsockopt(zmq.SUBSCRIBE, "")
        self._mgmt_sockets["worker_sub"].connect(mgmt_master_worker)

        # adress for the communication from worker(s) to master
        mgmt_worker_master = "inproc://worker/master/coordination/"

        # connect the worker with the master
        self._mgmt_sockets["worker_pub"] = self._ctx.socket(zmq.PUB)
        self._mgmt_sockets["worker_pub"].bind(mgmt_worker_master)
        sock = self._ctx.socket(zmq.SUB)
        sock.setsockopt(zmq.SUBSCRIBE, "")
        sock.connect(mgmt_worker_master)
        self._mgmt_sockets["tmp2"] = sock
        self._mgmt_sockets["master_sub"] = ZMQStream(sock, self._io_loop)

    def _setup_data_sockets(self):

        self._worker_sockets = dict()

        # address for master -> worker communication
        data_master_worker = "inproc://master/worker/pipeline/"

        sock = self._ctx.socket(zmq.PUSH)
        sock.bind(data_master_worker)
        self._worker_sockets["tmp3"] = sock
        self._worker_sockets["master_push"] = ZMQStream(sock, self._io_loop)
        self._worker_sockets["worker_pull"] = self._ctx.socket(zmq.PULL)
        self._worker_sockets["worker_pull"].connect(data_master_worker)

        # address for worker -> master communication
        data_worker_master = "inproc://worker/master/pipeline/"

        self._worker_sockets["worker_pub"] = self._ctx.socket(zmq.PUB)
        self._worker_sockets["worker_pub"].bind(data_worker_master)
        sock = self._ctx.socket(zmq.SUB)
        sock.setsockopt(zmq.SUBSCRIBE, "")
        sock.connect(data_worker_master)
        self._worker_sockets["tmp4"] = sock
        self._worker_sockets["master_sub"] = ZMQStream(sock, self._io_loop)

    def on_mgmt_end(self, _msg):
        self._io_loop.stop()
class ZmqTornadoIntegrationTest(unittest.TestCase):

    def setUp(self):

        # create the io_loop
        self._io_loop = IOLoop.instance()

        # and the context
        self._ctx = zmq.Context(1)

        self._settings = Settings()
        self._settings.ZEROMQ_MASTER_PUSH = 'inproc://spyder-zmq-master-push'
        self._settings.ZEROMQ_WORKER_PROC_FETCHER_PULL = \
            self._settings.ZEROMQ_MASTER_PUSH
        self._settings.ZEROMQ_MASTER_SUB = 'inproc://spyder-zmq-master-sub'
        self._settings.ZEROMQ_WORKER_PROC_EXTRACTOR_PUB = \
            self._settings.ZEROMQ_MASTER_SUB

        self._settings.ZEROMQ_MGMT_MASTER = 'inproc://spyder-zmq-mgmt-master'
        self._settings.ZEROMQ_MGMT_WORKER = 'inproc://spyder-zmq-mgmt-worker'

        # setup the mgmt sockets
        self._setup_mgmt_sockets()

        # setup the data sockets
        self._setup_data_servers()

        # setup the management interface
        self._mgmt = ZmqMgmt( self._mgmt_sockets['worker_sub'],
            self._mgmt_sockets['worker_pub'], io_loop=self._io_loop)
        self._mgmt.start()
        self._mgmt.add_callback(ZMQ_SPYDER_MGMT_WORKER, self.on_mgmt_end)

    def tearDown(self):
        # stop the mgmt
        self._mgmt.stop()

        # close all sockets
        for socket in self._mgmt_sockets.itervalues():
            socket.close()
        for socket in self._worker_sockets.itervalues():
            socket.close()

        # terminate the context
        self._ctx.term()

    def _setup_mgmt_sockets(self):

        self._mgmt_sockets = dict()

        # adress for the communication from master to worker(s)
        mgmt_master_worker = self._settings.ZEROMQ_MGMT_MASTER

        # connect the master with the worker
        # the master is a ZMQStream because we are sending msgs from the test
        sock = self._ctx.socket(zmq.PUB)
        sock.bind(mgmt_master_worker)
        self._mgmt_sockets['tmp1'] = sock
        self._mgmt_sockets['master_pub'] = ZMQStream(sock, self._io_loop)
        # the worker stream is created inside the ZmqMgmt class
        self._mgmt_sockets['worker_sub'] = self._ctx.socket(zmq.SUB)
        self._mgmt_sockets['worker_sub'].setsockopt(zmq.SUBSCRIBE, "")
        self._mgmt_sockets['worker_sub'].connect(mgmt_master_worker)

        # adress for the communication from worker(s) to master
        mgmt_worker_master = self._settings.ZEROMQ_MGMT_WORKER

        # connect the worker with the master
        self._mgmt_sockets['worker_pub'] = self._ctx.socket(zmq.PUB)
        self._mgmt_sockets['worker_pub'].bind(mgmt_worker_master)
        sock = self._ctx.socket(zmq.SUB)
        sock.setsockopt(zmq.SUBSCRIBE, "")
        sock.connect(mgmt_worker_master)
        self._mgmt_sockets['tmp2'] = sock
        self._mgmt_sockets['master_sub'] = ZMQStream(sock, self._io_loop)

    def _setup_data_servers(self):

        self._worker_sockets = dict()

        # address for master -> worker communication
        data_master_worker = self._settings.ZEROMQ_WORKER_PROC_EXTRACTOR_PULL

        sock = self._ctx.socket(zmq.PUSH)
        sock.bind(data_master_worker)
        self._worker_sockets['tmp3'] = sock
        self._worker_sockets['master_push'] = ZMQStream(sock, self._io_loop)

        # address for worker -> master communication
        data_worker_master = self._settings.ZEROMQ_WORKER_PROC_EXTRACTOR_PUB

        sock = self._ctx.socket(zmq.SUB)
        sock.setsockopt(zmq.SUBSCRIBE, "")
        sock.bind(data_worker_master)
        self._worker_sockets['tmp4'] = sock
        self._worker_sockets['master_sub'] = ZMQStream(sock, self._io_loop)

    def on_mgmt_end(self, _msg):
        self._io_loop.stop()