def _random_consume():
"""
Consume random queue
@return:
"""
qn = random.randint(0, config['consumers_max'])
queue = "q{}".format(qn)
events = [
"e{}".format(
qn % (config['consumers_max'] / config['publish_multiplicator']))
]
delete_queue_when_unused = True
if random.randint(0, 100) > 20:
delete_queue_when_unused = True
if random.randint(0, 100) > 80:
delete_queue_when_unused = random.randint(1, 5)
consumer_id = mqks.consume(
queue,
events,
_on_msg,
delete_queue_when_unused=delete_queue_when_unused)
state['consumers'][consumer_id] = dict(
delete_queue_when_unused=delete_queue_when_unused,
consumer_expired_time=None if config['consumer_ttl_seconds'] is None
else (time.time() + random.randint(*config['consumer_ttl_seconds'])))
def flow():
try:
queue = dtid()
consumer_id = mqks.consume(queue=queue, events=[queue], on_msg=lambda msg: None, delete_queue_when_unused=1, confirm=True)
for _ in xrange(10):
mqks.publish(queue, dtid())
time.sleep(1)
mqks.delete_consumer(consumer_id)
except Exception:
crit()
def test_abcd(self):
# Two consumers of the same queue - Alice and Bob:
alice = Mock()
mqks.consume('greetings', ['hi', 'hello'], alice, confirm=True)
bob = Mock()
mqks.consume('greetings', ['hi', 'hello'], bob, confirm=True)
# Consumer of another queue - Charlie:
charlie = Mock()
mqks.consume('greetings-and-byes', ['hi', 'hello', 'bye', 'good-bye'],
charlie,
confirm=True)
# One publish from Dave:
dave_msg_id = mqks.publish('hello', 'world', confirm=True)
# One and only one of Alice and Bob should get the message from Dave:
if not alice.called or bob.called:
gevent.sleep(0.1)
self.assertTrue(alice.called == 1 or bob.called == 1)
self.assertFalse(alice.called and bob.called)
on_msg = alice if alice.called else bob
self.assertEqual(len(on_msg.args), 1)
self.assertEqual(len(on_msg.kwargs), 0)
msg = on_msg.args[0]
self.assertIsInstance(msg, dict)
self.assertIn('id', msg)
self.assertEqual(msg['id'], dave_msg_id)
self.assertIn('event', msg)
self.assertEqual(msg['event'], 'hello')
self.assertIn('data', msg)
self.assertEqual(msg['data'], 'world')
# Charlie should get message from Dave too:
self.assertEqual(charlie.called, 1)
msg = charlie.args[0]
self.assertEqual(msg['id'], dave_msg_id)
self.assertEqual(msg['event'], 'hello')
self.assertEqual(msg['data'], 'world')
def test_rebind(self):
# Consume:
on_msg = Mock()
consumer_id = mqks.consume('q1', ['e1', 'e2', 'u.42.e1', 'u.43.e1'], on_msg, confirm=True)
# "replace" was tested in "test_update_consumer".
# Testing other args here:
mqks.rebind('q1', remove=['e2'], add=['e3', 'e4'], remove_mask=['u.*.e1'], confirm=True)
worker = mqks.get_worker('q1')
self.assertEqual(mqks.state['consumers'][worker][consumer_id], 'q1 e1 e3 e4')
# Cleanup:
mqks.delete_consumer(consumer_id, confirm=True)
mqks.delete_queue('q1', confirm=True)
def test_update_consumer(self):
# Consume:
on_msg = Mock()
on_disconnect = Mock()
on_reconnect = Mock()
consumer_id = mqks.consume('q1', ['e1', 'e2'],
on_msg,
on_disconnect=on_disconnect,
on_reconnect=on_reconnect,
delete_queue_when_unused=5.0,
manual_ack=True,
confirm=True)
worker = mqks.get_worker('q1')
self.assertEqual(
mqks.state['consumers'][worker][consumer_id],
'q1 e1 e2 --delete-queue-when-unused=5.0 --manual-ack')
# Rebind leads to "update consumer" flow:
mqks.rebind('q1', ['e2', 'e3'], confirm=True)
self.assertEqual(
mqks.state['consumers'][worker][consumer_id],
'q1 e2 e3 --delete-queue-when-unused=5.0 --manual-ack')
self.assertEqual(
mqks._eval("' '.join(sorted(state.events_by_queues['q1']))",
worker=worker), 'e2 e3')
# Reconnect:
old_client = mqks._eval("request['client']", worker=worker)
try:
mqks._eval("state.socks_by_clients[request['client']].shutdown(2)",
worker=worker,
timeout=1) # Disconnect.
except gevent.Timeout:
pass
gevent.sleep(2) # Reconnect.
self.assertEqual(on_disconnect.called, 1)
self.assertEqual(on_reconnect.called, 1)
# consumer_id is updated:
self.assertEqual(on_reconnect.args[0], consumer_id)
self.assertNotEqual(on_reconnect.args[1], consumer_id)
consumer_id = on_reconnect.args[1]
# client id is updated:
new_client = mqks._eval("request['client']", worker=worker)
self.assertNotEqual(old_client, new_client)
# Reconsume used last consumer config:
self.assertEqual(
mqks._eval("' '.join(sorted(state.events_by_queues['q1']))",
worker=worker), 'e2 e3')
# Message delivery is still working after reconnect:
self.assertEqual(on_msg.called, 0)
mqks.publish('e3', 'd1', confirm=True)
if not on_msg.called:
gevent.sleep(0.1)
self.assertEqual(on_msg.called, 1)
self.assertEqual(on_msg.args[0]['data'], 'd1')
# Add events:
mqks.delete_consumer(consumer_id, confirm=True)
consumer_id = mqks.consume('q1', ['e3', 'e4'],
on_msg,
delete_queue_when_unused=5.0,
manual_ack=True,
add_events=True,
confirm=True)
self.assertEqual(
mqks.state['consumers'][worker][consumer_id],
'q1 e2 e3 e4 --delete-queue-when-unused=5.0 --manual-ack')
self.assertEqual(
mqks._eval("' '.join(sorted(state.events_by_queues['q1']))",
worker=worker), 'e2 e3 e4')
# Cleanup:
mqks.delete_consumer(consumer_id, confirm=True)
mqks.delete_queue('q1', confirm=True)
def test_manual_ack(self):
# Consume:
on_msg = Mock()
consumer_id = mqks.consume('q1', ['e1'],
on_msg,
manual_ack=True,
confirm=True)
# Publish:
mqks.publish('e1', 'd1', confirm=True)
# Original message:
if not on_msg.called:
gevent.sleep(0.1)
self.assertEqual(on_msg.called, 1)
msg = on_msg.args[0]
self.assertEqual(msg['data'], 'd1')
self.assertNotIn('retry', msg)
# Delete consumer without ack, consume again:
mqks.delete_consumer(consumer_id, confirm=True)
consumer_id = mqks.consume('q1', ['e1'],
on_msg,
manual_ack=True,
confirm=True)
# Retried message:
if on_msg.called == 1:
gevent.sleep(0.1)
self.assertEqual(on_msg.called, 2)
msg = on_msg.args[0]
self.assertEqual(msg['data'], 'd1')
self.assertEqual(msg['retry'], '1')
# Ack, delete_consumer, consume, no retries:
msg['ack'](confirm=True)
mqks.delete_consumer(consumer_id, confirm=True)
consumer_id = mqks.consume('q1', ['e1'],
on_msg,
manual_ack=True,
confirm=True)
gevent.sleep(0.5)
self.assertEqual(on_msg.called, 2)
# Publish, get original, reject, no other consumers, get retry:
mqks.publish('e1', 'd2', confirm=True)
if on_msg.called == 2:
gevent.sleep(0.1)
self.assertEqual(on_msg.called, 3)
msg = on_msg.args[0]
self.assertEqual(msg['data'], 'd2')
self.assertNotIn('retry', msg)
msg['reject'](confirm=True)
if on_msg.called == 3:
gevent.sleep(0.1)
self.assertEqual(on_msg.called, 4)
msg = on_msg.args[0]
self.assertEqual(msg['data'], 'd2')
self.assertEqual(msg['retry'], '1')
# Reject all, no other consumers, get retry:
mqks.reject_all(consumer_id, confirm=True)
if on_msg.called == 4:
gevent.sleep(0.1)
self.assertEqual(on_msg.called, 5)
msg = on_msg.args[0]
self.assertEqual(msg['data'], 'd2')
self.assertEqual(msg['retry'], '2')
# Ack all, delete_consumer, consume, no retries:
mqks.ack_all(consumer_id, confirm=True)
mqks.delete_consumer(consumer_id, confirm=True)
consumer_id = mqks.consume('q1', ['e1'],
on_msg,
manual_ack=True,
confirm=True)
gevent.sleep(0.5)
self.assertEqual(on_msg.called, 5)
# Cleanup:
mqks.delete_consumer(consumer_id, confirm=True)
mqks.delete_queue('q1', confirm=True)