def test_single_socket_forwarder_bind(self): dev = devices.ThreadDevice(zmq.FORWARDER, zmq.REP, -1) req = self.context.socket(zmq.REQ) port = 12345 req.connect('tcp://127.0.0.1:%i'%port) dev.bind_in('tcp://127.0.0.1:%i'%port) dev.start() time.sleep(.25) msg = asbytes('hello') req.send(msg) self.assertEquals(msg, req.recv()) del dev del req dev = devices.ThreadDevice(zmq.FORWARDER, zmq.REP, -1) req = self.context.socket(zmq.REQ) port = 12346 req.connect('tcp://127.0.0.1:%i'%port) dev.bind_in('tcp://127.0.0.1:%i'%port) dev.start() time.sleep(.25) msg = asbytes('hello again') req.send(msg) self.assertEquals(msg, req.recv()) del dev del req
def test_prefix(self): if zmq.zmq_version() < '3.0.0': raise SkipTest("Only applies to libzmq >= 3.0") xrep, xreq = self.create_bound_pair(zmq.XREP, zmq.XREQ) msg = [asbytes(p) for p in 'hi there'.split()] xreq.send_multipart(msg) recvd = xrep.recv_multipart() self.assertTrue(isinstance(recvd, tuple)) self.assertEquals(len(recvd), 2) prefix, real = recvd self.assertTrue(isinstance(prefix, list)) self.assertEquals(len(prefix), 1) self.assertEquals(real, msg) xrep.send_multipart(real, prefix=prefix) echo = xreq.recv_multipart() self.assertTrue(isinstance(echo, list)) self.assertEquals(echo, real) extra = [asbytes('pre')] xrep.send_multipart(msg, prefix=prefix + extra) recvd = xreq.recv_multipart() self.assertTrue(isinstance(recvd, tuple)) self.assertEquals(len(recvd), 2) prefix, real = recvd self.assertTrue(isinstance(prefix, list)) self.assertEquals(len(prefix), 1) self.assertEquals(prefix, extra) self.assertEquals(real, msg)
def test_init_iface(self): logger = self.logger ctx = self.context handler = handlers.PUBHandler(self.iface) self.assertFalse(handler.ctx is ctx) self.sockets.append(handler.socket) # handler.ctx.term() handler = handlers.PUBHandler(self.iface, self.context) self.sockets.append(handler.socket) self.assertTrue(handler.ctx is ctx) handler.setLevel(logging.DEBUG) handler.root_topic = self.topic logger.addHandler(handler) sub = ctx.socket(zmq.SUB) self.sockets.append(sub) sub.setsockopt(zmq.SUBSCRIBE, self.topic) sub.connect(self.iface) import time time.sleep(0.25) msg1 = 'message' logger.info(msg1) (topic, msg2) = sub.recv_multipart() self.assertEquals(topic, asbytes('zmq.INFO')) self.assertEquals(msg2, asbytes(msg1 + '\n')) logger.removeHandler(handler)
def test_lifecycle1(self): """Run through a ref counting cycle with a copy.""" for i in range(5, 16): # 32, 64,..., 65536 s = (2**i)*x rc = 2 self.assertEquals(grc(s), rc) m = zmq.Message(s) rc += 2 self.assertEquals(grc(s), rc) m2 = copy.copy(m) rc += 1 self.assertEquals(grc(s), rc) b = m2.buffer rc += view_rc self.assertEquals(grc(s), rc) self.assertEquals(s, asbytes(str(m))) self.assertEquals(s, asbytes(m2)) self.assertEquals(s, m.bytes) # self.assert_(s is str(m)) # self.assert_(s is str(m2)) del m2 rc -= 1 self.assertEquals(grc(s), rc) rc -= view_rc del b self.assertEquals(grc(s), rc) del m rc -= 2 self.assertEquals(grc(s), rc) self.assertEquals(rc, 2) del s
def test_lifecycle2(self): """Run through a different ref counting cycle with a copy.""" for i in range(5, 16): # 32, 64,..., 65536 s = (2**i)*x rc = 2 self.assertEquals(grc(s), rc) m = zmq.Message(s) rc += 2 self.assertEquals(grc(s), rc) m2 = copy.copy(m) rc += 1 self.assertEquals(grc(s), rc) b = m.buffer rc += view_rc self.assertEquals(grc(s), rc) self.assertEquals(s, asbytes(str(m))) self.assertEquals(s, asbytes(m2)) self.assertEquals(s, m2.bytes) self.assertEquals(s, m.bytes) # self.assert_(s is str(m)) # self.assert_(s is str(m2)) del b self.assertEquals(grc(s), rc) del m # m.buffer is kept until m is del'd rc -= view_rc rc -= 1 self.assertEquals(grc(s), rc) del m2 rc -= 2 self.assertEquals(grc(s), rc) self.assertEquals(rc, 2) del s
def build_device(self, mon_sub=asbytes(""), in_prefix=asbytes('in'), out_prefix=asbytes('out')): if zmq.zmq_version() >= '3': raise SkipTest( "MonitoredQueues don't work reliably on libzmq >= 3.0.0") self.device = devices.ThreadMonitoredQueue(zmq.PAIR, zmq.PAIR, zmq.PUB, in_prefix, out_prefix) alice = self.context.socket(zmq.PAIR) bob = self.context.socket(zmq.PAIR) mon = self.context.socket(zmq.SUB) aport = alice.bind_to_random_port('tcp://127.0.0.1') bport = bob.bind_to_random_port('tcp://127.0.0.1') mport = mon.bind_to_random_port('tcp://127.0.0.1') mon.setsockopt(zmq.SUBSCRIBE, mon_sub) self.device.connect_in("tcp://127.0.0.1:%i" % aport) self.device.connect_out("tcp://127.0.0.1:%i" % bport) self.device.connect_mon("tcp://127.0.0.1:%i" % mport) time.sleep(.2) self.device.start() self.sockets.extend([alice, bob, mon]) return alice, bob, mon
def build_device(self, mon_sub=asbytes(""), in_prefix=asbytes('in'), out_prefix=asbytes('out')): self.device = devices.ThreadMonitoredQueue(zmq.PAIR, zmq.PAIR, zmq.PUB, in_prefix, out_prefix) alice = self.context.socket(zmq.PAIR) bob = self.context.socket(zmq.PAIR) mon = self.context.socket(zmq.SUB) aport = alice.bind_to_random_port('tcp://127.0.0.1') bport = bob.bind_to_random_port('tcp://127.0.0.1') mport = mon.bind_to_random_port('tcp://127.0.0.1') mon.setsockopt(zmq.SUBSCRIBE, mon_sub) self.device.connect_in("tcp://127.0.0.1:%i" % aport) self.device.connect_out("tcp://127.0.0.1:%i" % bport) self.device.connect_mon("tcp://127.0.0.1:%i" % mport) self.device.start() time.sleep(.2) try: # this is currenlty necessary to ensure no dropped monitor messages # see LIBZMQ-248 for more info mon.recv_multipart(zmq.NOBLOCK) except zmq.ZMQError: pass self.sockets.extend([alice, bob, mon]) return alice, bob, mon
def test_lifecycle2(self): """Run through a different ref counting cycle with a copy.""" for i in range(5, 16): # 32, 64,..., 65536 s = (2**i)*x rc = 2 self.assertEquals(grc(s), rc) m = zmq.Frame(s) rc += 2 self.assertEquals(grc(s), rc) m2 = copy.copy(m) rc += 1 self.assertEquals(grc(s), rc) b = m.buffer rc += view_rc self.assertEquals(grc(s), rc) self.assertEquals(s, asbytes(str(m))) self.assertEquals(s, asbytes(m2)) self.assertEquals(s, m2.bytes) self.assertEquals(s, m.bytes) # self.assert_(s is str(m)) # self.assert_(s is str(m2)) del b self.assertEquals(grc(s), rc) del m # m.buffer is kept until m is del'd rc -= view_rc rc -= 1 self.assertEquals(grc(s), rc) del m2 rc -= 2 self.assertEquals(grc(s), rc) self.assertEquals(rc, 2) del s
def test_prefix(self): if zmq.zmq_version() < '3.0.0': raise SkipTest("Only applies to libzmq >= 3.0") xrep, xreq = self.create_bound_pair(zmq.XREP, zmq.XREQ) msg = [ asbytes(p) for p in 'hi there'.split() ] xreq.send_multipart(msg) recvd = xrep.recv_multipart() self.assertTrue(isinstance(recvd, tuple)) self.assertEquals(len(recvd), 2) prefix, real = recvd self.assertTrue(isinstance(prefix, list)) self.assertEquals(len(prefix), 1) self.assertEquals(real, msg) xrep.send_multipart(real, prefix=prefix) echo = xreq.recv_multipart() self.assertTrue(isinstance(echo, list)) self.assertEquals(echo, real) extra = [asbytes('pre')] xrep.send_multipart(msg, prefix=prefix+extra) recvd = xreq.recv_multipart() self.assertTrue(isinstance(recvd, tuple)) self.assertEquals(len(recvd), 2) prefix, real = recvd self.assertTrue(isinstance(prefix, list)) self.assertEquals(len(prefix), 1) self.assertEquals(prefix, extra) self.assertEquals(real, msg)
def test_pubsub(self): s1, s2 = self.create_bound_pair(zmq.PUB, zmq.SUB) s2.setsockopt(zmq.SUBSCRIBE, asbytes('')) # Sleep to allow sockets to connect. wait() poller = zmq.Poller() poller.register(s1, zmq.POLLIN|zmq.POLLOUT) poller.register(s2, zmq.POLLIN) # Now make sure that both are send ready. socks = dict(poller.poll()) self.assertEquals(socks[s1], zmq.POLLOUT) self.assertEquals(s2 in socks, 0) # Make sure that s1 stays in POLLOUT after a send. s1.send(asbytes('msg1')) socks = dict(poller.poll()) self.assertEquals(socks[s1], zmq.POLLOUT) # Make sure that s2 is POLLIN after waiting. wait() socks = dict(poller.poll()) self.assertEquals(socks[s2], zmq.POLLIN) # Make sure that s2 goes into 0 after recv. s2.recv() socks = dict(poller.poll()) self.assertEquals(s2 in socks, 0) poller.unregister(s1) poller.unregister(s2) # Wait for everything to finish. wait()
def build_device(self, mon_sub=asbytes(""), in_prefix=asbytes('in'), out_prefix=asbytes('out')): self.device = devices.ThreadMonitoredQueue(zmq.PAIR, zmq.PAIR, zmq.PUB, in_prefix, out_prefix) alice = self.context.socket(zmq.PAIR) bob = self.context.socket(zmq.PAIR) mon = self.context.socket(zmq.SUB) aport = alice.bind_to_random_port('tcp://127.0.0.1') bport = bob.bind_to_random_port('tcp://127.0.0.1') mport = mon.bind_to_random_port('tcp://127.0.0.1') mon.setsockopt(zmq.SUBSCRIBE, mon_sub) self.device.connect_in("tcp://127.0.0.1:%i"%aport) self.device.connect_out("tcp://127.0.0.1:%i"%bport) self.device.connect_mon("tcp://127.0.0.1:%i"%mport) self.device.start() time.sleep(.2) try: # this is currenlty necessary to ensure no dropped monitor messages # see LIBZMQ-248 for more info mon.recv_multipart(zmq.NOBLOCK) except zmq.ZMQError: pass self.sockets.extend([alice, bob, mon]) return alice, bob, mon
def test_pair(self): s1, s2 = self.create_bound_pair(zmq.PAIR, zmq.PAIR) # Sleep to allow sockets to connect. wait() poller = zmq.Poller() poller.register(s1, zmq.POLLIN|zmq.POLLOUT) poller.register(s2, zmq.POLLIN|zmq.POLLOUT) # Poll result should contain both sockets socks = dict(poller.poll()) # Now make sure that both are send ready. self.assertEquals(socks[s1], zmq.POLLOUT) self.assertEquals(socks[s2], zmq.POLLOUT) # Now do a send on both, wait and test for zmq.POLLOUT|zmq.POLLIN s1.send(asbytes('msg1')) s2.send(asbytes('msg2')) wait() socks = dict(poller.poll()) self.assertEquals(socks[s1], zmq.POLLOUT|zmq.POLLIN) self.assertEquals(socks[s2], zmq.POLLOUT|zmq.POLLIN) # Make sure that both are in POLLOUT after recv. s1.recv() s2.recv() socks = dict(poller.poll()) self.assertEquals(socks[s1], zmq.POLLOUT) self.assertEquals(socks[s2], zmq.POLLOUT) poller.unregister(s1) poller.unregister(s2) # Wait for everything to finish. wait()
def test_init_iface(self): logger = self.logger ctx = self.context handler = handlers.PUBHandler(self.iface) self.assertFalse(handler.ctx is ctx) self.sockets.append(handler.socket) # handler.ctx.term() handler = handlers.PUBHandler(self.iface, self.context) self.sockets.append(handler.socket) self.assertTrue(handler.ctx is ctx) handler.setLevel(logging.DEBUG) handler.root_topic = self.topic logger.addHandler(handler) sub = ctx.socket(zmq.SUB) self.sockets.append(sub) sub.setsockopt(zmq.SUBSCRIBE, self.topic) sub.connect(self.iface) import time; time.sleep(0.25) msg1 = 'message' logger.info(msg1) (topic, msg2) = sub.recv_multipart() self.assertEquals(topic, asbytes('zmq.INFO')) self.assertEquals(msg2, asbytes(msg1+'\n')) logger.removeHandler(handler)
def test_single_socket_forwarder_bind(self): dev = devices.ThreadDevice(zmq.QUEUE, zmq.REP, -1) # select random port: binder = self.context.socket(zmq.REQ) port = binder.bind_to_random_port('tcp://127.0.0.1') binder.close() time.sleep(0.1) req = self.context.socket(zmq.REQ) req.connect('tcp://127.0.0.1:%i' % port) dev.bind_in('tcp://127.0.0.1:%i' % port) dev.start() time.sleep(.25) msg = asbytes('hello') req.send(msg) self.assertEquals(msg, req.recv()) del dev req.close() dev = devices.ThreadDevice(zmq.QUEUE, zmq.REP, -1) # select random port: binder = self.context.socket(zmq.REQ) port = binder.bind_to_random_port('tcp://127.0.0.1') binder.close() time.sleep(0.1) req = self.context.socket(zmq.REQ) req.connect('tcp://127.0.0.1:%i' % port) dev.bind_in('tcp://127.0.0.1:%i' % port) dev.start() time.sleep(.25) msg = asbytes('hello again') req.send(msg) self.assertEquals(msg, req.recv()) del dev req.close()
def test_lifecycle1(self): """Run through a ref counting cycle with a copy.""" for i in range(5, 16): # 32, 64,..., 65536 s = (2**i) * x rc = 2 self.assertEquals(grc(s), rc) m = zmq.Frame(s) rc += 2 self.assertEquals(grc(s), rc) m2 = copy.copy(m) rc += 1 self.assertEquals(grc(s), rc) b = m2.buffer rc += view_rc self.assertEquals(grc(s), rc) self.assertEquals(s, asbytes(str(m))) self.assertEquals(s, asbytes(m2)) self.assertEquals(s, m.bytes) # self.assert_(s is str(m)) # self.assert_(s is str(m2)) del m2 rc -= 1 self.assertEquals(grc(s), rc) rc -= view_rc del b self.assertEquals(grc(s), rc) del m rc -= 2 self.assertEquals(grc(s), rc) self.assertEquals(rc, 2) del s
def test_router_router(self): """test router-router MQ devices""" if zmq.zmq_version() >= "4.0.0": raise SkipTest("Only for libzmq < 4") dev = devices.ThreadMonitoredQueue(zmq.ROUTER, zmq.ROUTER, zmq.PUB, "in", "out") dev.setsockopt_in(zmq.LINGER, 0) dev.setsockopt_out(zmq.LINGER, 0) dev.setsockopt_mon(zmq.LINGER, 0) binder = self.context.socket(zmq.DEALER) porta = binder.bind_to_random_port("tcp://127.0.0.1") portb = binder.bind_to_random_port("tcp://127.0.0.1") binder.close() time.sleep(0.1) a = self.context.socket(zmq.DEALER) a.identity = asbytes("a") b = self.context.socket(zmq.DEALER) b.identity = asbytes("b") a.connect("tcp://127.0.0.1:%i" % porta) dev.bind_in("tcp://127.0.0.1:%i" % porta) b.connect("tcp://127.0.0.1:%i" % portb) dev.bind_out("tcp://127.0.0.1:%i" % portb) dev.start() time.sleep(0.2) msg = [asbytes(m) for m in ("hello", "there")] a.send_multipart(["b"] + msg) bmsg = self.recv_multipart(b) self.assertEquals(bmsg, ["a"] + msg) b.send_multipart(bmsg) amsg = self.recv_multipart(a) self.assertEquals(amsg, ["b"] + msg)
def test_single_socket_forwarder_bind(self): dev = devices.ThreadDevice(zmq.QUEUE, zmq.REP, -1) # select random port: binder = self.context.socket(zmq.REQ) port = binder.bind_to_random_port('tcp://127.0.0.1') binder.close() time.sleep(0.1) req = self.context.socket(zmq.REQ) req.connect('tcp://127.0.0.1:%i'%port) dev.bind_in('tcp://127.0.0.1:%i'%port) dev.start() time.sleep(.25) msg = asbytes('hello') req.send(msg) self.assertEquals(msg, self.recv(req)) del dev req.close() dev = devices.ThreadDevice(zmq.QUEUE, zmq.REP, -1) # select random port: binder = self.context.socket(zmq.REQ) port = binder.bind_to_random_port('tcp://127.0.0.1') binder.close() time.sleep(0.1) req = self.context.socket(zmq.REQ) req.connect('tcp://127.0.0.1:%i'%port) dev.bind_in('tcp://127.0.0.1:%i'%port) dev.start() time.sleep(.25) msg = asbytes('hello again') req.send(msg) self.assertEquals(msg, self.recv(req)) del dev req.close()
def test_basic(self): s1, s2 = self.create_bound_pair(zmq.PUB, zmq.SUB) s2.setsockopt(zmq.SUBSCRIBE,asbytes('')) time.sleep(0.1) msg1 = asbytes('message') s1.send(msg1) msg2 = s2.recv() # This is blocking! self.assertEquals(msg1, msg2)
def test_basic(self): s1, s2 = self.create_bound_pair(zmq.PUB, zmq.SUB) s2.setsockopt(zmq.SUBSCRIBE, asbytes('')) time.sleep(0.1) msg1 = asbytes('message') s1.send(msg1) msg2 = s2.recv() # This is blocking! self.assertEquals(msg1, msg2)
def _process_socks(self, socks): if self.result_collector in socks: data = self.result_collector.recv_string() if 'status' not in data: self.stats_handler.write_result(ujson.loads(data)) self.external_publisher.send_multipart([asbytes(self.topic), asbytes(data)]) self.messages += 1 else: self.turrets_manager.process_message(ujson.loads(data))
def test_close(self): ctx = zmq.Context() s = ctx.socket(zmq.PUB) s.close() self.assertRaises(zmq.ZMQError, s.bind, asbytes('')) self.assertRaises(zmq.ZMQError, s.connect, asbytes('')) self.assertRaises(zmq.ZMQError, s.setsockopt, zmq.SUBSCRIBE, asbytes('')) self.assertRaises(zmq.ZMQError, s.send, asbytes('asdf')) self.assertRaises(zmq.ZMQError, s.recv) del ctx
def test_bad_send_recv(self): s1, s2 = self.create_bound_pair(zmq.REQ, zmq.REP) for copy in (True,False): self.assertRaisesErrno(zmq.EFSM, s1.recv, copy=copy) self.assertRaisesErrno(zmq.EFSM, s2.send, asbytes('asdf'), copy=copy) # I have to have this or we die on an Abort trap. msg1 = asbytes('asdf') msg2 = self.ping_pong(s1, s2, msg1) self.assertEquals(msg1, msg2)
def topic_pub(self, port=5556, topic="test", data="test data"): ''' usage: python pub_and_sub_tool.py topic_pub --data=hahah ''' pub = context.socket(zmq.PUB) pub.bind(f"tcp://127.0.0.1:{port}") time.sleep(0.5) # import IPython;IPython.embed() pub.send_multipart([asbytes(topic), asbytes(data)]) print("topic: {} ; msg: {}".format(topic, data))
def test_reply(self): alice, bob, mon = self.build_device() alices = asbytes("hello bob").split() alice.send_multipart(alices) bobs = self.recv_multipart(bob) self.assertEquals(alices, bobs) bobs = asbytes("hello alice").split() bob.send_multipart(bobs) alices = self.recv_multipart(alice) self.assertEquals(alices, bobs) self.teardown_device()
def test_topic(self): s1, s2 = self.create_bound_pair(zmq.PUB, zmq.SUB) s2.setsockopt(zmq.SUBSCRIBE, asbytes('x')) time.sleep(0.1) msg1 = asbytes('message') s1.send(msg1) self.assertRaisesErrno(zmq.EAGAIN, s2.recv, zmq.NOBLOCK) msg1 = asbytes('xmessage') s1.send(msg1) msg2 = s2.recv() self.assertEquals(msg1, msg2)
def monitoredqueue(frontend_url="tcp://*:4444", server_url="tcp://localhost:5555", capture_url="tcp://*:7777"): mondev = MonitoredQueue(zmq.ROUTER, zmq.DEALER, zmq.PUB, asbytes("in"), asbytes("out")) mondev.bind_in(frontend_url) mondev.connect_out(server_url) mondev.bind_mon(capture_url) mondev.setsockopt_in(zmq.SNDHWM, 1) mondev.start() print("Monitored queue started")
def test_bad_sockopts(self): """Test that appropriate errors are raised on bad socket options""" s = self.context.socket(zmq.PUB) self.sockets.append(s) s.setsockopt(zmq.LINGER, 0) # unrecognized int sockopts pass through to libzmq, and should raise EINVAL self.assertRaisesErrno(zmq.EINVAL, s.setsockopt, 9999, 5) self.assertRaisesErrno(zmq.EINVAL, s.getsockopt, 9999) # but only int sockopts are allowed through this way, otherwise raise a TypeError self.assertRaises(TypeError, s.setsockopt, 9999, asbytes("5")) # some sockopts are valid in general, but not on every socket: self.assertRaisesErrno(zmq.EINVAL, s.setsockopt, zmq.SUBSCRIBE, asbytes('hi'))
def test_tracker(self): "test the MessageTracker object for tracking when zmq is done with a buffer" addr = 'tcp://127.0.0.1' a = self.context.socket(zmq.PUB) port = a.bind_to_random_port(addr) a.close() iface = "%s:%i" % (addr, port) a = self.context.socket(zmq.PAIR) # a.setsockopt(zmq.IDENTITY, asbytes("a")) b = self.context.socket(zmq.PAIR) self.sockets.extend([a, b]) a.connect(iface) time.sleep(0.1) p1 = a.send(asbytes('something'), copy=False, track=True) self.assertTrue(isinstance(p1, zmq.MessageTracker)) self.assertFalse(p1.done) p2 = a.send_multipart(list(map(asbytes, ['something', 'else'])), copy=False, track=True) self.assert_(isinstance(p2, zmq.MessageTracker)) self.assertEquals(p2.done, False) self.assertEquals(p1.done, False) b.bind(iface) msg = b.recv_multipart() self.assertEquals(p1.done, True) self.assertEquals(msg, (list(map(asbytes, ['something'])))) msg = b.recv_multipart() self.assertEquals(p2.done, True) self.assertEquals(msg, list(map(asbytes, ['something', 'else']))) m = zmq.Message(asbytes("again"), track=True) self.assertEquals(m.tracker.done, False) p1 = a.send(m, copy=False) p2 = a.send(m, copy=False) self.assertEquals(m.tracker.done, False) self.assertEquals(p1.done, False) self.assertEquals(p2.done, False) msg = b.recv_multipart() self.assertEquals(m.tracker.done, False) self.assertEquals(msg, list(map(asbytes, ['again']))) msg = b.recv_multipart() self.assertEquals(m.tracker.done, False) self.assertEquals(msg, list(map(asbytes, ['again']))) self.assertEquals(p1.done, False) self.assertEquals(p2.done, False) pm = m.tracker del m time.sleep(0.1) self.assertEquals(p1.done, True) self.assertEquals(p2.done, True) m = zmq.Message(asbytes('something'), track=False) self.assertRaises(ValueError, a.send, m, copy=False, track=True)
def monitordevice(): in_prefix=asbytes('in') out_prefix=asbytes('out') monitoringdevice = MonitoredQueue(zmq.XREP, zmq.XREQ, zmq.PUB, in_prefix, out_prefix) monitoringdevice.bind_in("tcp://127.0.0.1:%d" % frontend_port) monitoringdevice.bind_out("tcp://127.0.0.1:%d" % backend_port) monitoringdevice.bind_mon("tcp://127.0.0.1:%d" % monitor_port) monitoringdevice.setsockopt_in(zmq.HWM, 1) monitoringdevice.setsockopt_out(zmq.HWM, 1) monitoringdevice.start() print "Program: Monitoring device has started"
def test_bad_send_recv(self): s1, s2 = self.create_bound_pair(zmq.REQ, zmq.REP) for copy in (True, False): self.assertRaisesErrno(zmq.EFSM, s1.recv, copy=copy) self.assertRaisesErrno(zmq.EFSM, s2.send, asbytes('asdf'), copy=copy) # I have to have this or we die on an Abort trap. msg1 = asbytes('asdf') msg2 = self.ping_pong(s1, s2, msg1) self.assertEquals(msg1, msg2)
def test_tracker(self): "test the MessageTracker object for tracking when zmq is done with a buffer" addr = 'tcp://127.0.0.1' a = self.context.socket(zmq.XREQ) port = a.bind_to_random_port(addr) a.close() del a iface = "%s:%i"%(addr,port) a = self.context.socket(zmq.XREQ) a.setsockopt(zmq.IDENTITY, asbytes("a")) b = self.context.socket(zmq.XREP) self.sockets.extend([a,b]) a.connect(iface) time.sleep(0.1) p1 = a.send(asbytes('something'), copy=False, track=True) self.assertTrue(isinstance(p1, zmq.MessageTracker)) self.assertFalse(p1.done) p2 = a.send_multipart(list(map(asbytes, ['something', 'else'])), copy=False, track=True) self.assert_(isinstance(p2, zmq.MessageTracker)) self.assertEquals(p2.done, False) self.assertEquals(p1.done, False) b.bind(iface) msg = b.recv_multipart() self.assertEquals(p1.done, True) self.assertEquals(msg, (list(map(asbytes, ['a', 'something'])))) msg = b.recv_multipart() self.assertEquals(p2.done, True) self.assertEquals(msg, list(map(asbytes, ['a', 'something', 'else']))) m = zmq.Message(asbytes("again"), track=True) self.assertEquals(m.done, False) p1 = a.send(m, copy=False) p2 = a.send(m, copy=False) self.assertEquals(m.done, False) self.assertEquals(p1.done, False) self.assertEquals(p2.done, False) msg = b.recv_multipart() self.assertEquals(m.done, False) self.assertEquals(msg, list(map(asbytes, ['a', 'again']))) msg = b.recv_multipart() self.assertEquals(m.done, False) self.assertEquals(msg, list(map(asbytes, ['a', 'again']))) self.assertEquals(p1.done, False) self.assertEquals(p2.done, False) pm = m.tracker del m time.sleep(0.1) self.assertEquals(p1.done, True) self.assertEquals(p2.done, True) m = zmq.Message(asbytes('something'), track=False) self.assertRaises(ValueError, a.send, m, copy=False, track=True)
def test_multi_tracker(self): m = zmq.Message(asbytes('asdf'), track=True) m2 = zmq.Message(asbytes('whoda'), track=True) mt = zmq.MessageTracker(m,m2) self.assertFalse(m.done) self.assertFalse(mt.done) self.assertRaises(zmq.NotDone, mt.wait, 0.1) del m time.sleep(0.1) self.assertRaises(zmq.NotDone, mt.wait, 0.1) self.assertFalse(mt.done) del m2 self.assertTrue(mt.wait() is None) self.assertTrue(mt.done)
def test_monitor(self): alice, bob, mon = self.build_device() alices = asbytes("hello bob").split() alice.send_multipart(alices) alices2 = asbytes("hello again").split() alice.send_multipart(alices2) alices3 = asbytes("hello again and again").split() alice.send_multipart(alices3) bobs = self.recv_multipart(bob) self.assertEquals(alices, bobs) mons = self.recv_multipart(mon) self.assertEquals([asbytes('in')] + bobs, mons) bobs = self.recv_multipart(bob) self.assertEquals(alices2, bobs) bobs = self.recv_multipart(bob) self.assertEquals(alices3, bobs) mons = self.recv_multipart(mon) self.assertEquals([asbytes('in')] + alices2, mons) bobs = asbytes("hello alice").split() bob.send_multipart(bobs) alices = self.recv_multipart(alice) self.assertEquals(alices, bobs) mons = self.recv_multipart(mon) self.assertEquals([asbytes('in')] + alices3, mons) mons = self.recv_multipart(mon) self.assertEquals([asbytes('out')] + bobs, mons) self.teardown_device()
def test_multi_tracker(self): m = zmq.Frame(asbytes('asdf'), track=True) m2 = zmq.Frame(asbytes('whoda'), track=True) mt = zmq.MessageTracker(m,m2) self.assertFalse(m.tracker.done) self.assertFalse(mt.done) self.assertRaises(zmq.NotDone, mt.wait, 0.1) del m time.sleep(0.1) self.assertRaises(zmq.NotDone, mt.wait, 0.1) self.assertFalse(mt.done) del m2 self.assertTrue(mt.wait() is None) self.assertTrue(mt.done)
def test_router_router(self): """test router-router MQ devices""" if zmq.zmq_version() >= '4.0.0': raise SkipTest("Only for libzmq < 4") dev = devices.ThreadMonitoredQueue(zmq.ROUTER, zmq.ROUTER, zmq.PUB, asbytes('in'), asbytes('out')) dev.setsockopt_in(zmq.LINGER, 0) dev.setsockopt_out(zmq.LINGER, 0) dev.setsockopt_mon(zmq.LINGER, 0) binder = self.context.socket(zmq.DEALER) porta = binder.bind_to_random_port('tcp://127.0.0.1') portb = binder.bind_to_random_port('tcp://127.0.0.1') binder.close() time.sleep(0.1) a = self.context.socket(zmq.DEALER) a.identity = asbytes('a') b = self.context.socket(zmq.DEALER) b.identity = asbytes('b') a.connect('tcp://127.0.0.1:%i' % porta) dev.bind_in('tcp://127.0.0.1:%i' % porta) b.connect('tcp://127.0.0.1:%i' % portb) dev.bind_out('tcp://127.0.0.1:%i' % portb) dev.start() time.sleep(0.2) msg = [asbytes(m) for m in ('hello', 'there')] a.send_multipart([asbytes('b')] + msg) bmsg = self.recv_multipart(b) self.assertEquals(bmsg, [asbytes('a')] + msg) b.send_multipart(bmsg) amsg = self.recv_multipart(a) self.assertEquals(amsg, [asbytes('b')] + msg)
def test_monitor(self): alice, bob, mon = self.build_device() alices = asbytes("hello bob").split() alice.send_multipart(alices) alices2 = asbytes("hello again").split() alice.send_multipart(alices2) alices3 = asbytes("hello again and again").split() alice.send_multipart(alices3) bobs = self.recv_multipart(bob) self.assertEquals(alices, bobs) mons = self.recv_multipart(mon) self.assertEquals([asbytes('in')]+bobs, mons) bobs = self.recv_multipart(bob) self.assertEquals(alices2, bobs) bobs = self.recv_multipart(bob) self.assertEquals(alices3, bobs) mons = self.recv_multipart(mon) self.assertEquals([asbytes('in')]+alices2, mons) bobs = asbytes("hello alice").split() bob.send_multipart(bobs) alices = self.recv_multipart(alice) self.assertEquals(alices, bobs) mons = self.recv_multipart(mon) self.assertEquals([asbytes('in')]+alices3, mons) mons = self.recv_multipart(mon) self.assertEquals([asbytes('out')]+bobs, mons) self.teardown_device()
def test_reqrep(self): s1, s2 = self.create_bound_pair(zmq.REP, zmq.REQ) # Sleep to allow sockets to connect. wait() poller = zmq.Poller() poller.register(s1, zmq.POLLIN|zmq.POLLOUT) poller.register(s2, zmq.POLLIN|zmq.POLLOUT) # Make sure that s1 is in state 0 and s2 is in POLLOUT socks = dict(poller.poll()) self.assertEquals(s1 in socks, 0) self.assertEquals(socks[s2], zmq.POLLOUT) # Make sure that s2 goes immediately into state 0 after send. s2.send(asbytes('msg1')) socks = dict(poller.poll()) self.assertEquals(s2 in socks, 0) # Make sure that s1 goes into POLLIN state after a time.sleep(). time.sleep(0.5) socks = dict(poller.poll()) self.assertEquals(socks[s1], zmq.POLLIN) # Make sure that s1 goes into POLLOUT after recv. s1.recv() socks = dict(poller.poll()) self.assertEquals(socks[s1], zmq.POLLOUT) # Make sure s1 goes into state 0 after send. s1.send(asbytes('msg2')) socks = dict(poller.poll()) self.assertEquals(s1 in socks, 0) # Wait and then see that s2 is in POLLIN. time.sleep(0.5) socks = dict(poller.poll()) self.assertEquals(socks[s2], zmq.POLLIN) # Make sure that s2 is in POLLOUT after recv. s2.recv() socks = dict(poller.poll()) self.assertEquals(socks[s2], zmq.POLLOUT) poller.unregister(s1) poller.unregister(s2) # Wait for everything to finish. wait()
def monitordevice(): in_prefix = asbytes('in') out_prefix = asbytes('out') monitoringdevice = MonitoredQueue(zmq.XREP, zmq.XREQ, zmq.PUB, in_prefix, out_prefix) monitoringdevice.bind_in("tcp://127.0.0.1:%d" % frontend_port) monitoringdevice.bind_out("tcp://127.0.0.1:%d" % backend_port) monitoringdevice.bind_mon("tcp://127.0.0.1:%d" % monitor_port) monitoringdevice.setsockopt_in(zmq.HWM, 1) monitoringdevice.setsockopt_out(zmq.HWM, 1) monitoringdevice.start() print "Program: Monitoring device has started"
def test_tracker(self): m = zmq.Frame(asbytes('asdf'), track=True) self.assertFalse(m.tracker.done) pm = zmq.MessageTracker(m) self.assertFalse(pm.done) del m self.assertTrue(pm.done)
def test_labels(self): """test device support for SNDLABEL""" if zmq.zmq_version() < '3.0.0': raise SkipTest("Only for libzmq 3") dev = devices.ThreadDevice(zmq.QUEUE, zmq.XREP, -1) # select random port: binder = self.context.socket(zmq.XREQ) port = binder.bind_to_random_port('tcp://127.0.0.1') binder.close() time.sleep(0.1) req = self.context.socket(zmq.REQ) req.connect('tcp://127.0.0.1:%i'%port) dev.bind_in('tcp://127.0.0.1:%i'%port) dev.start() time.sleep(.25) msg = asbytes('hello') req.send(msg, zmq.SNDLABEL) req.send(msg, zmq.SNDMORE) req.send(msg) self.assertEquals(msg, self.recv(req)) self.assertTrue(req.rcvlabel) self.assertEquals(msg, self.recv(req)) self.assertTrue(req.rcvmore) self.assertEquals(msg, self.recv(req)) del dev req.close()
def test_large_msg(self): s1, s2 = self.create_bound_pair(zmq.REQ, zmq.REP) msg1 = asbytes(10000*'X') for i in range(10): msg2 = self.ping_pong(s1, s2, msg1) self.assertEquals(msg1, msg2)
def test_term_hang(self): rep,req = self.create_bound_pair(zmq.ROUTER, zmq.DEALER) req.setsockopt(zmq.LINGER, 0) req.send(asbytes('hello'), copy=False) req.close() rep.close() self.context.term()
def manage_workflows(self): """Manages the workflows to be executed and the workers. It waits for the server to submit a request to execute a workflow, and then passes the workflow off to an available worker, once one becomes available. """ while True: if self.thread_exit: break # There is a worker available and a workflow in the queue, so pop it off and send it to the worker if self.available_workers and not self.pending_workflows.empty(): workflow = self.pending_workflows.get() worker = self.available_workers.pop() self.workflow_comms[workflow['execution_uid']] = worker self.request_socket.send_multipart( [worker, b"", asbytes(json.dumps(workflow))]) # If there is a worker available but no pending workflows, then see if there are any other workers # available, but do not block in case a workflow becomes available else: try: worker, empty, ready = self.request_socket.recv_multipart( flags=zmq.NOBLOCK) if ready == b"Ready" or ready == b"Done": self.available_workers.append(worker) except zmq.ZMQError: gevent.sleep(0.1) continue self.request_socket.close() self.comm_socket.close() return
def test_multiple(self): s1, s2 = self.create_bound_pair(zmq.REQ, zmq.REP) for i in range(10): msg1 = asbytes(i*' ') msg2 = self.ping_pong(s1, s2, msg1) self.assertEquals(msg1, msg2)
def export(self, name, columns, points): """Write the points to the ZeroMQ server.""" logger.debug("Export {} stats to ZeroMQ".format(name)) # Create DB input data = dict(zip(columns, points)) # Do not publish empty stats if data == {}: return False # Glances envelopes the stats in a publish message with two frames: # - First frame containing the following prefix (STRING) # - Second frame with the Glances plugin name (STRING) # - Third frame with the Glances plugin stats (JSON) message = [b(self.prefix), b(name), asbytes(json.dumps(data))] # Write data to the ZeroMQ bus # Result can be view: tcp://host:port try: self.client.send_multipart(message) except Exception as e: logger.error("Cannot export {} stats to ZeroMQ ({})".format( name, e)) return True
def wrapper(func): for table in tables: # hash ring for cache update queues = [Queue() for _ in range(workers)] hash_ring = ketama.Continuum() for q in queues: hash_ring[str(hash(q))] = q self.update_queues[table] = hash_ring cache_update = self._cache_update_gen(table, func, multi=multi) self.workers[table] = [ Worker("%s_cache_update" % table, q, cache_update, multi=multi, logger_name="%s.%s" % (self.name, table)) for q in queues ] # single worker for cache delete delete_q = Queue() self.delete_queues[table] = delete_q cache_delete = self._cache_delete_gen(table) self.workers[table].append( Worker("%s_cache_delete" % table, delete_q, cache_delete, multi=True, logger_name="%s.%s" % (self.name, table))) self.socket.setsockopt(zmq.SUBSCRIBE, asbytes(table)) return func
def export(self, name, columns, points): """Write the points to the ZeroMQ server.""" logger.debug("Export {} stats to ZeroMQ".format(name)) # Create DB input data = dict(zip(columns, points)) # Do not publish empty stats if data == {}: return False # Glances envelopes the stats in a publish message with two frames: # - First frame containing the following prefix (STRING) # - Second frame with the Glances plugin name (STRING) # - Third frame with the Glances plugin stats (JSON) message = [b(self.prefix), b(name), asbytes(json.dumps(data))] # Write data to the ZeroMQ bus # Result can be view: tcp://host:port try: self.client.send_multipart(message) except Exception as e: logger.error("Cannot export {} stats to ZeroMQ ({})".format(name, e)) return True
def test_root_topic(self): logger, handler, sub = self.connect_handler() handler.socket.bind(self.iface) sub2 = sub.context.socket(zmq.SUB) self.sockets.append(sub2) sub2.connect(self.iface) sub2.setsockopt(zmq.SUBSCRIBE, asbytes('')) handler.root_topic = asbytes('twoonly') msg1 = 'ignored' logger.info(msg1) self.assertRaisesErrno(zmq.EAGAIN, sub.recv, zmq.NOBLOCK) topic, msg2 = sub2.recv_multipart() self.assertEquals(topic, asbytes('twoonly.INFO')) self.assertEquals(msg2, asbytes(msg1 + '\n')) logger.removeHandler(handler)
def wrapper(func): for table in tables: # hash ring for cache update queues = [Queue() for _ in range(workers)] hash_ring = ConsistentHashRing() for q in queues: hash_ring[hash(q)] = q self.update_queues[table] = hash_ring cache_update = self._cache_update_gen(table, func, multi=multi) self.workers[table] = [ Worker("%s_cache_update" % table, q, cache_update, multi=multi, logger_name="%s.%s" % (self.name, table)) for q in queues] # single worker for cache delete delete_q = Queue() self.delete_queues[table] = delete_q cache_delete = self._cache_delete_gen(table) self.workers[table].append( Worker("%s_cache_delete" % table, delete_q, cache_delete, multi=True, logger_name="%s.%s" % (self.name, table))) self.socket.setsockopt(zmq.SUBSCRIBE, asbytes(table)) return func
def test_labels(self): """test device support for SNDLABEL""" raise SkipTest("LABELs have been removed") dev = devices.ThreadDevice(zmq.QUEUE, zmq.XREP, -1) # select random port: binder = self.context.socket(zmq.XREQ) port = binder.bind_to_random_port('tcp://127.0.0.1') binder.close() time.sleep(0.1) req = self.context.socket(zmq.REQ) req.connect('tcp://127.0.0.1:%i'%port) dev.bind_in('tcp://127.0.0.1:%i'%port) dev.start() time.sleep(.25) msg = asbytes('hello') req.send(msg, zmq.SNDLABEL) req.send(msg, zmq.SNDMORE) req.send(msg) self.assertEquals(msg, self.recv(req)) self.assertTrue(req.rcvlabel) self.assertEquals(msg, self.recv(req)) self.assertTrue(req.rcvmore) self.assertEquals(msg, self.recv(req)) del dev req.close()
def test_recv_multipart(self): a,b = self.create_bound_pair() msg = asbytes('hi') for i in range(3): a.send(msg) time.sleep(0.1) for i in range(3): self.assertEquals(b.recv_multipart(), [msg])