def test_proxied_fanout(collector, worker_pub, socket, device,
topic, addr1, addr2):
# customer |----| forwarder with XPUB/XSUB |---> | worker
# collector | <-----------------------------------|
# run forwarder
forwarder_in_addr, forwarder_out_addr = addr1, addr2
device(socket(zmq.XSUB), socket(zmq.XPUB),
forwarder_in_addr, forwarder_out_addr)
# connect to the devices
worker_sub1 = socket(zmq.SUB)
worker_sub2 = socket(zmq.SUB)
worker_sub1.set(zmq.SUBSCRIBE, topic)
worker_sub2.set(zmq.SUBSCRIBE, topic)
worker_sub1.connect(forwarder_out_addr)
worker_sub2.connect(forwarder_out_addr)
pub = socket(zmq.PUB)
pub.connect(forwarder_in_addr)
sync_pubsub(pub, [worker_sub1, worker_sub2], topic)
# make and start workers
app = Application()
worker1 = zeronimo.Worker(app, [worker_sub1], worker_pub)
worker2 = zeronimo.Worker(app, [worker_sub2], worker_pub)
fanout = zeronimo.Fanout(pub, collector)
with running([worker1, worker2, collector]):
# zeronimo!
results = fanout.emit(topic, 'zeronimo')
assert get_results(results) == ['zeronimo', 'zeronimo']
def test_shared_client_port(socket, addr1, addr2, addr3, addr4,
reply_sockets, fin):
if not addr1.startswith('tcp://'):
pytest.skip()
# sockets for workers
pull1 = socket(zmq.PULL)
pull2 = socket(zmq.PULL)
pull1.bind(addr1)
pull2.bind(addr2)
reply_sock = socket(zmq.PUB)
reply_sock.bind(addr3)
# spawn workers
app = Application()
worker1 = zeronimo.Worker(app, [pull1], reply_sock, info='worker1')
worker2 = zeronimo.Worker(app, [pull2], reply_sock, info='worker2')
worker1.start()
worker2.start()
fin(worker1.stop)
fin(worker2.stop)
# sockets for collectors
reply_topic1 = rand_str()
reply_topic2 = rand_str()
collector_sock1 = socket(zmq.SUB)
collector_sock2 = socket(zmq.SUB)
collector_sock1.set(zmq.SUBSCRIBE, reply_topic1)
collector_sock2.set(zmq.SUBSCRIBE, reply_topic2)
collector_sock1.connect(addr3)
collector_sock2.connect(addr3)
sync_pubsub(reply_sock, [collector_sock1], reply_topic1)
sync_pubsub(reply_sock, [collector_sock2], reply_topic2)
# make collectors
collector1 = zeronimo.Collector(collector_sock1, reply_topic1)
collector2 = zeronimo.Collector(collector_sock2, reply_topic2)
# sockets for customers
push1 = socket(zmq.PUSH)
push2 = socket(zmq.PUSH)
push1.connect(addr4 + ';' + addr1[len('tcp://'):])
push2.connect(addr4 + ';' + addr2[len('tcp://'):])
# e.g., 'tcp://127.0.0.1:44444;127.0.0.1:11111'
# make customers
customer1 = zeronimo.Customer(push1, collector1)
customer2 = zeronimo.Customer(push2, collector2)
# do test
for x in range(100):
r = customer1.call('add', 0, 1)
assert r.get() == 1
assert r.worker_info == worker1.info
r = customer2.call('add', 0, 2)
assert r.get() == 2
assert r.worker_info == worker2.info
def test_pair_with_collector(socket, addr, reply_sockets):
# Failed at 0.3.0.
left = socket(zmq.PAIR)
right = socket(zmq.PAIR)
left.bind(addr)
right.connect(addr)
reply_sock, (collector_sock, reply_topic) = reply_sockets()
worker = zeronimo.Worker(Application(), [left], reply_sock)
sync_pubsub(reply_sock, [collector_sock], reply_topic)
collector = zeronimo.Collector(collector_sock, reply_topic)
customer = zeronimo.Customer(right, collector)
with running([worker, collector]):
assert customer.call('zeronimo').get() == 'zeronimo'
def test_proxied_collector(worker, push, socket, device, addr1, addr2, fin):
# customer |-------------------> | worker
# collector | <---| forwarder |---|
device(socket(zmq.XSUB), socket(zmq.XPUB), addr1, addr2)
reply_topic = rand_str()
collector_sock = socket(zmq.SUB)
collector_sock.set(zmq.SUBSCRIBE, reply_topic)
collector_sock.connect(addr2)
worker.reply_socket.connect(addr1)
sync_pubsub(worker.reply_socket, [collector_sock], reply_topic)
collector = zeronimo.Collector(collector_sock, reply_topic)
customer = zeronimo.Customer(push, collector)
assert customer.call('zeronimo').get() == 'zeronimo'
def test_fanout_by_other_types(left_type, right_type, socket, addr1, addr2):
fanout_sock = socket(left_type)
worker_sock = socket(right_type)
worker_sock.bind(addr1)
fanout_sock.connect(addr1)
worker_pub, collector_sub = socket(zmq.PUB), socket(zmq.SUB)
worker_pub.bind(addr2)
collector_sub.connect(addr2)
collector_sub.set(zmq.SUBSCRIBE, 'xxx')
sync_pubsub(worker_pub, [collector_sub], 'xxx')
worker = zeronimo.Worker(Application(), [worker_sock], worker_pub)
collector = zeronimo.Collector(collector_sub, 'xxx')
fanout = zeronimo.Fanout(fanout_sock, collector)
with running([worker]):
assert get_results(fanout.emit('anything', 'zeronimo')) == ['zeronimo']
def test_device(collector, worker_pub, socket, device,
topic, addr1, addr2, addr3, addr4, fin):
# customer |-----| forwarder |---> | worker
# | <----| streamer |-----|
# run streamer
streamer_in_addr, streamer_out_addr = addr1, addr2
forwarder_in_addr, forwarder_out_addr = addr3, addr4
device(socket(zmq.PULL), socket(zmq.PUSH),
streamer_in_addr, streamer_out_addr)
# run forwarder
sub = socket(zmq.SUB)
sub.set(zmq.SUBSCRIBE, b'')
device(sub, socket(zmq.PUB), forwarder_in_addr, forwarder_out_addr)
# connect to the devices
worker_pull1 = socket(zmq.PULL)
worker_pull2 = socket(zmq.PULL)
worker_pull1.connect(streamer_out_addr)
worker_pull2.connect(streamer_out_addr)
worker_sub1 = socket(zmq.SUB)
worker_sub2 = socket(zmq.SUB)
worker_sub1.set(zmq.SUBSCRIBE, topic)
worker_sub2.set(zmq.SUBSCRIBE, topic)
worker_sub1.connect(forwarder_out_addr)
worker_sub2.connect(forwarder_out_addr)
push = socket(zmq.PUSH)
push.connect(streamer_in_addr)
pub = socket(zmq.PUB)
pub.connect(forwarder_in_addr)
sync_pubsub(pub, [worker_sub1, worker_sub2], topic)
# make and start workers
app = Application()
worker1 = zeronimo.Worker(app, [worker_pull1, worker_sub1], worker_pub)
worker2 = zeronimo.Worker(app, [worker_pull2, worker_sub2], worker_pub)
worker1.start()
worker2.start()
fin(worker1.stop)
fin(worker2.stop)
# zeronimo!
customer = zeronimo.Customer(push, collector)
fanout = zeronimo.Fanout(pub, collector)
assert customer.call('zeronimo').get() == 'zeronimo'
assert \
get_results(fanout.emit(topic, 'zeronimo')) == \
['zeronimo', 'zeronimo']
def test_drop_not_subscribed_topic(socket, worker, collector, addr, topic):
xpub = socket(zmq.XPUB)
xpub.bind(addr)
sub = [s for s in worker.sockets if s.type == zmq.SUB][0]
sub.connect(addr)
msg = xpub.recv()
sync_pubsub(xpub, [sub], topic)
assert msg.startswith(b'\x01')
found_topic = msg[1:]
assert found_topic == topic
packed = []
def pack(x):
packed.append(x)
return zeronimo.messaging.PACK(x)
fanout = zeronimo.Fanout(xpub, collector, pack=pack,
drop_if=lambda x: x != found_topic)
assert get_results(fanout.emit(topic[::-1], 'zeronimo')) == []
assert len(packed) == 0
assert get_results(fanout.emit(topic, 'zeronimo')) == ['zeronimo']
assert len(packed) == 1
def test_subscription(worker1, worker2, collector, pub, topic):
fanout = zeronimo.Fanout(pub, collector)
sub1 = [sock for sock in worker1.sockets if sock.type == zmq.SUB][0]
sub2 = [sock for sock in worker2.sockets if sock.type == zmq.SUB][0]
sub1.set(zmq.UNSUBSCRIBE, topic)
assert len(fanout.emit(topic, 'zeronimo')) == 1
sub2.set(zmq.UNSUBSCRIBE, topic)
assert fanout.emit(topic, 'zeronimo') == []
sub1.set(zmq.SUBSCRIBE, topic)
# sync_pubsub will disturb the worker. so the worker should be stopped
# during sync_pubsub works.
worker1.stop()
sync_pubsub(pub, [sub1], topic)
worker1.start()
assert len(fanout.emit(topic, 'zeronimo')) == 1
sub2.set(zmq.SUBSCRIBE, topic)
# same reason
worker1.stop()
worker2.stop()
sync_pubsub(pub, [sub1, sub2], topic)
worker1.start()
worker2.start()
assert len(fanout.emit(topic, 'zeronimo')) == 2