def test_client_defaults(self):
id, name = rand_int(), rand_string()
client = Client(id, name)
self.assertEquals(client.id, id)
self.assertEquals(client.name, name)
self.assertEquals(client.is_active, True)
def test_client_custom_attrs(self):
id, name, is_active = rand_int(), rand_string(), rand_bool()
client = Client(id, name, is_active)
self.assertEquals(client.id, id)
self.assertEquals(client.name, name)
self.assertEquals(client.is_active, is_active)
def test_delete_from_topic_no_subscribers(self):
""" Tests deleting of messages from a topic without subscribers.
"""
ps = RedisPubSub(self.kvdb, self.key_prefix)
msg_value = '"msg_value"'
topic = Topic('/test/delete')
ps.add_topic(topic)
producer = Client('Producer', 'producer')
ps.add_producer(producer, topic)
pub_ctx = PubCtx()
pub_ctx.client_id = producer.id
pub_ctx.topic = topic.name
pub_ctx.msg = Message(msg_value)
msg_id = ps.publish(pub_ctx).msg.msg_id
# We've got a msg_id now and we know this is a topic with no subsribers. In that case, deleting a messages should
# also delete all the metadata related to it along with its payload.
# First, let's confirm the messages is actually there
result = self.kvdb.zrange(ps.MSG_IDS_PREFIX.format(topic.name), 0, -1)
eq_(result, [msg_id])
result = self.kvdb.hgetall(ps.MSG_VALUES_KEY)
eq_(len(result), 1)
self.assertTrue(msg_id in result)
result = self.kvdb.hgetall(ps.MSG_METADATA_KEY)
eq_(len(result), 1)
self.assertTrue(msg_id in result)
result = self.kvdb.hgetall(ps.MSG_EXPIRE_AT_KEY)
eq_(len(result), 1)
self.assertTrue(msg_id in result)
# Ok, now delete the message and confirm it's not in Redis anymore
ps.delete_from_topic(topic.name, msg_id)
result = self.kvdb.zrange(ps.MSG_IDS_PREFIX.format(topic.name), 0, -1)
eq_(result, [])
result = self.kvdb.hgetall(ps.MSG_VALUES_KEY)
eq_({}, result)
result = self.kvdb.hgetall(ps.MSG_METADATA_KEY)
eq_({}, result)
result = self.kvdb.hgetall(ps.MSG_EXPIRE_AT_KEY)
eq_({}, result)
def test_subscribe(self):
client_id, client_name = rand_int(), rand_string()
client = Client(client_id, client_name)
topics = rand_string(rand_int())
sub_key = self.api.subscribe(client.id, topics)
self.assertEquals(self.api.impl.sub_to_cons[sub_key], client_id)
self.assertEquals(self.api.impl.cons_to_sub[client_id], sub_key)
self.assertEquals(sorted(self.api.impl.cons_to_topic[client_id]), sorted(topics))
for topic in topics:
self.assertIn(client_id, self.api.impl.topic_to_cons[topic])
def test_default_clients(self):
# Initially, default clients are dummy ones.
default_consumer = self.api.get_default_consumer()
default_producer = self.api.get_default_producer()
self.assertEquals(default_consumer.id, None)
self.assertEquals(default_consumer.name, None)
self.assertEquals(default_consumer.is_active, True)
self.assertEquals(default_producer.id, None)
self.assertEquals(default_producer.name, None)
self.assertEquals(default_producer.is_active, True)
cons_id = rand_int()
cons_name = rand_string()
cons_is_active = rand_bool()
prod_name = rand_string()
prod_id = rand_int()
prod_is_active = rand_bool()
cons = Client(cons_id, cons_name, cons_is_active)
prod = Client(prod_id, prod_name, prod_is_active)
self.api.set_default_consumer(cons)
self.api.set_default_producer(prod)
default_consumer = self.api.get_default_consumer()
default_producer = self.api.get_default_producer()
self.assertEquals(default_consumer.id, cons_id)
self.assertEquals(default_consumer.name, cons_name)
self.assertEquals(default_consumer.is_active, cons_is_active)
self.assertEquals(default_producer.id, prod_id)
self.assertEquals(default_producer.name, prod_name)
self.assertEquals(default_producer.is_active, prod_is_active)
def _publish_move(self, move=True, **kwargs):
payload = rand_string()
topic = Topic(rand_string())
self.api.add_topic(topic)
producer = Client(rand_int(), rand_string())
self.api.add_producer(producer, topic)
ctx = self.api.publish(payload,
topic.name,
client_id=producer.id,
**kwargs)
if move:
self.api.impl.move_to_target_queues()
return payload, topic, producer, ctx
def test_publish_exceptions(self):
payload = rand_string()
producer = Client(rand_int(), rand_string())
def invoke_publish(payload, topic, producer_id):
self.api.publish(payload, topic, client_id=producer_id)
# KeyError because no such producer is in self.api.impl.producers.
self.assertRaises(KeyError, invoke_publish, payload, rand_string(),
producer.id)
# Adding a producer but still, no such topic.
self.api.add_producer(producer, Topic(rand_string()))
self.assertRaises(PubSubException, invoke_publish, payload,
rand_string(), producer.id)
# Adding a topic but still PubSubException is raised because the producer is not allowed to use it.
topic = Topic(rand_string())
self.api.add_topic(topic)
self.assertRaises(PubSubException, invoke_publish, payload, topic.name,
producer.id)
# Combining the topic and producer, no exception is raised now.
self.api.add_producer(producer, topic)
invoke_publish(payload, topic.name, producer.id)
# But it's not possible to publish to inactive topics.
self.api.impl.topics[topic.name].is_active = False
self.assertRaises(PubSubException, invoke_publish, payload, topic.name,
producer.id)
# Make the topic active and it can be published to again.
self.api.impl.topics[topic.name].is_active = True
invoke_publish(payload, topic.name, producer.id)
# Inactive producers cannot publish to topics either.
self.api.impl.producers[producer.id].is_active = False
self.assertRaises(PubSubException, invoke_publish, payload, topic.name,
producer.id)
# Making them active means they can publish again.
self.api.impl.producers[producer.id].is_active = True
invoke_publish(payload, topic.name, producer.id)
def test_get_reject_acknowledge(self):
client_id, client_name = rand_int(), rand_string()
payload, topic, producer, ctx = self._publish_move(
move=False, add_consumer=True, client_id=client_id, client_name=client_name)
client = Client(client_id, client_name)
sub_key = self.api.subscribe(client.id, topic.name)
# Moves a message to the consumer's queue
self.api.impl.move_to_target_queues()
self._check_consumer_queue_before_get(ctx, sub_key)
# Consumer gets a message which puts it in the in-flight state.
self._check_get(ctx, sub_key, topic, producer, client)
# However, there should be nothing in the consumer's queue.
consumer_msg_ids = self.kvdb.lrange(self.api.impl.CONSUMER_MSG_IDS_PREFIX.format(sub_key), 0, -1)
self.assertEquals(consumer_msg_ids, [])
# Consumer rejects the message which puts it back on a queue.
self.api.reject(sub_key, ctx.msg.msg_id)
# After rejection it's as though the message has just been published.
self._check_consumer_queue_before_get(ctx, sub_key)
# Get after rejection works as before.
self._check_get(ctx, sub_key, topic, producer, client)
# Consumer acknowledges a message.
self.api.acknowledge(sub_key, ctx.msg.msg_id)
keys = self.kvdb.keys('{}*'.format(self.key_prefix))
self.assertEquals(len(keys), 3)
now = datetime.utcnow()
last_pub_time = parse(self.kvdb.hgetall(self.api.impl.LAST_PUB_TIME_KEY)[topic.name])
last_seen_consumer = parse(self.kvdb.hgetall(self.api.impl.LAST_SEEN_CONSUMER_KEY)[str(client.id)])
last_seen_producer = parse(self.kvdb.hgetall(self.api.impl.LAST_SEEN_PRODUCER_KEY)[str(producer.id)])
self.assertTrue(last_pub_time < now, 'last_pub_time:`{}` is not less than now:`{}`'.format(last_pub_time, now))
self.assertTrue(last_seen_consumer < now, 'last_seen_consumer:`{}` is not less than now:`{}`'.format(last_seen_consumer, now))
self.assertTrue(last_seen_producer < now, 'last_seen_producer:`{}` is not less than now:`{}`'.format(last_seen_producer, now))
def _publish_move(self, move=True, **kwargs):
payload = rand_string()
topic = Topic(rand_string())
self.api.add_topic(topic)
client_id = kwargs.pop('client_id', rand_int())
client_name = kwargs.pop('client_name', rand_string())
if kwargs.pop('add_consumer', False):
consumer = Consumer(client_id, client_name)
self.api.add_consumer(consumer, topic)
producer = Client(client_id, client_name)
self.api.add_producer(producer, topic)
ctx = self.api.publish(payload, topic.name, client_id=producer.id, **kwargs)
if move:
self.api.impl.move_to_target_queues()
return payload, topic, producer, ctx
def test_ack_delete_from(self):
""" Tests ack and delete on a consumer's queue.
"""
# Both ack and delete use the same implementation and it's only the user-exposed API that differs.
ps = RedisPubSub(self.kvdb, self.key_prefix)
msg_value = '"msg_value"'
topic = Topic('/test/delete')
ps.add_topic(topic)
producer = Client('Producer', 'producer')
ps.add_producer(producer, topic)
consumer = Consumer('Consumer', 'consumer', sub_key=new_cid())
ps.add_consumer(consumer, topic)
pub_ctx = PubCtx()
pub_ctx.client_id = producer.id
pub_ctx.topic = topic.name
pub_ctx.msg = Message(msg_value)
msg_id = ps.publish(pub_ctx).msg.msg_id
ps.move_to_target_queues()
# A message has been published and move to a consumer's queue so let's confirm the fact first.
result = self.kvdb.lrange(
ps.CONSUMER_MSG_IDS_PREFIX.format(consumer.sub_key), 0, -1)
eq_(result, [msg_id])
# Ok, now delete the message and confirm it's not in the consumer's queue anymore.
ps.delete_from_consumer_queue(consumer.sub_key, [msg_id])
result = self.kvdb.lrange(
ps.CONSUMER_MSG_IDS_PREFIX.format(consumer.sub_key), 0, -1)
eq_(result, [])
def test_get_callback_consumers(self):
ps = RedisPubSub(self.kvdb, self.key_prefix)
topic = Topic('/test/delete')
ps.add_topic(topic)
producer = Client('Producer', 'producer')
ps.add_producer(producer, topic)
id1 = 'Consumer CB1'
name1 = 'consumer-cb1'
sub_key1 = new_cid()
callback_id1 = rand_int()
id2 = 'Consumer CB2'
name2 = 'consumer-cb2'
sub_key2 = new_cid()
callback_id2 = rand_int()
consumer_cb1 = Consumer(
id1,
name1,
sub_key=sub_key1,
delivery_mode=PUB_SUB.DELIVERY_MODE.CALLBACK_URL.id,
callback_id=callback_id1)
consumer_cb2 = Consumer(
id2,
name2,
sub_key=sub_key2,
delivery_mode=PUB_SUB.DELIVERY_MODE.CALLBACK_URL.id,
callback_id=callback_id2)
consumer_pull = Consumer('Consumer pull',
'consumer-pull',
sub_key=new_cid(),
delivery_mode=PUB_SUB.DELIVERY_MODE.PULL.id)
consumer_inactive = Consumer(
'Consumer pull',
'consumer-pull',
is_active=False,
sub_key=new_cid(),
delivery_mode=PUB_SUB.DELIVERY_MODE.PULL.id)
ps.add_consumer(consumer_cb1, topic)
ps.add_consumer(consumer_cb2, topic)
ps.add_consumer(
consumer_pull,
topic) # This one should not be returned because it's a pull one
ps.add_consumer(
consumer_inactive,
topic) # This one should not be returned because it's inactive
consumers = list(ps.get_callback_consumers())
# Only 2 are returned, the rest won't make it
eq_(len(consumers), 2)
# Sort by each consumer's ID, i.e. in lexicographical order
consumers.sort(key=attrgetter('id'))
consumer1 = consumers[0]
eq_(consumer1.id, id1)
eq_(consumer1.name, name1)
eq_(consumer1.is_active, True)
eq_(consumer1.sub_key, sub_key1)
eq_(consumer1.callback_id, callback_id1)
consumer2 = consumers[1]
eq_(consumer2.id, id2)
eq_(consumer2.name, name2)
eq_(consumer2.is_active, True)
eq_(consumer2.sub_key, sub_key2)
eq_(consumer2.callback_id, callback_id2)
def test_delete_from_topic_has_subscribers(self):
""" Tests deleting of messages from a topic which has subscribers.
"""
ps = RedisPubSub(self.kvdb, self.key_prefix)
msg_value = '"msg_value"'
topic = Topic('/test/delete')
ps.add_topic(topic)
producer = Client('Producer', 'producer')
ps.add_producer(producer, topic)
consumer = Consumer('Consumer', 'consumer', new_cid())
ps.add_consumer(consumer, topic)
pub_ctx = PubCtx()
pub_ctx.client_id = producer.id
pub_ctx.topic = topic.name
pub_ctx.msg = Message(msg_value)
msg_id = ps.publish(pub_ctx).msg.msg_id
# We've got a msg_id now and we know this is a topic which has a subscriber so deleting a message from the topic
# should not delete it from any other place because the consumer may still hold onto it.
# First, let's confirm the messages is actually there
result = self.kvdb.zrange(ps.MSG_IDS_PREFIX.format(topic.name), 0, -1)
eq_(result, [msg_id])
result = self.kvdb.hgetall(ps.MSG_VALUES_KEY)
eq_(len(result), 1)
self.assertTrue(msg_id in result)
result = self.kvdb.hgetall(ps.MSG_METADATA_KEY)
eq_(len(result), 1)
self.assertTrue(msg_id in result)
result = self.kvdb.hgetall(ps.MSG_EXPIRE_AT_KEY)
eq_(len(result), 1)
self.assertTrue(msg_id in result)
# Ok, now delete the message and confirm it's not in the topic anymore however it's still kept elsewhere.
ps.delete_from_topic(topic.name, msg_id)
result = self.kvdb.zrange(ps.MSG_IDS_PREFIX.format(topic.name), 0, -1)
eq_(result, [])
result = self.kvdb.hgetall(ps.MSG_VALUES_KEY)
eq_(len(result), 1)
self.assertTrue(msg_id in result)
result = self.kvdb.hgetall(ps.MSG_METADATA_KEY)
eq_(len(result), 1)
self.assertTrue(msg_id in result)
result = self.kvdb.hgetall(ps.MSG_EXPIRE_AT_KEY)
eq_(len(result), 1)
self.assertTrue(msg_id in result)
def test_full_path(self):
""" Tests full sub/pub/ack/reject path with 4 topics and 3 clients. Doesn't test background tasks.
"""
# Give up early if there is no connection to Redis at all
if not self.has_redis:
return
""" What is tested.
- 3 clients connect to pub/sub: CRM, Billing and ERP.
- 4 topics are created: /cust/new, /cust/update, /adsl/new and /adsl/update
- Subscriptions are:
- CRM subs to /adsl/new
- CRM subs to /adsl/update
- Billing subs to /cust/new
- Billing subs to /cust/update
- ERP subs to /adsl/new
- ERP subs to /adsl/update
- Publications are:
- CRM publishes Msg-CRM1 to /cust/new -------------- TTL of 0.1s
- CRM publishes Msg-CRM2 to /cust/update ----------- TTL of 0.2s
- Billing publishes Msg-Billing1 to /adsl/new ------ TTL of 0.3s
- Billing publishes Msg-Billing2 to /adsl/update --- TTL of 3600s
- (ERP doesn't publish anything)
- Expected deliveries are:
- Msg-CRM1 goes to Billing
- Msg-CRM2 goes to Billing
- Msg-Billing1 goes to CRM
- Msg-Billing2 goes to CRM
- Msg-Billing1 goes to ERP
- Msg-Billing2 goes to ERP
- Confirmations are:
- CRM acks Msg-Billing1
- CRM rejects Msg-Billing2
- Billing acks Msg-CRM1
- Billing acks Msg-CRM2
- ERP rejects Msg-Billing1
- ERP acks Msg-Billing2
- Clean up tasks are
- Msg-CRM1 is deleted because it's confirmed by its only recipient of Billing
- Msg-CRM2 is deleted because it's confirmed by its only recipient of Billing
- Msg-Billing1 is deleted because:
- CRM confirms it
- ERP rejects it but the message's TTL is 0.3s so it times out
(In real world ERP would have possibly acknowledged it in say, 2s, but it would've been too late)
- Msg-Billing2 is not deleted because:
- ERP confirms it
- CRM rejects it and the message's TTL is 3600s so it's still around when a clean up task runs
"""
ps = RedisPubSub(self.kvdb, self.key_prefix)
msg_billing1_value = '"msg_billing1"'
msg_billing2_value = '"msg_billing2"'
msg_crm1_value = '"msg_crm1"'
msg_crm2_value = '"msg_crm2"'
# Check all the Lua programs are loaded
eq_(len(ps.lua_programs), 9)
for attr in dir(ps):
if attr.startswith('LUA'):
value = getattr(ps, attr)
self.assertTrue(value in ps.lua_programs, value)
topic_cust_new = Topic('/cust/new')
topic_cust_update = Topic('/cust/update')
topic_adsl_new = Topic('/adsl/new')
topic_adsl_update = Topic('/adsl/update')
ps.add_topic(topic_cust_new)
ps.add_topic(topic_cust_update)
ps.add_topic(topic_adsl_new)
ps.add_topic(topic_adsl_update)
# Check all the topics are cached locally
eq_(len(ps.topics), 4)
for topic in (topic_cust_new, topic_cust_update, topic_adsl_new,
topic_adsl_update):
eq_(ps.topics[topic.name], topic)
client_id_crm = Client('CRM', 'CRM')
client_id_billing = Client('Billing', 'Billing')
client_id_erp = Client('ERP', 'ERP')
ps.add_producer(client_id_crm, topic_cust_new)
ps.add_producer(client_id_crm, topic_cust_update)
ps.add_producer(client_id_billing, topic_adsl_new)
ps.add_producer(client_id_billing, topic_adsl_update)
# Check producers have been registered for topics
eq_(len(ps.prod_to_topic), 2)
self.assertTrue(client_id_crm.id in ps.prod_to_topic)
self.assertTrue(client_id_billing.id in ps.prod_to_topic)
self.assertTrue(client_id_erp.id not in ps.prod_to_topic)
self.assertTrue(isinstance(ps.prod_to_topic[client_id_crm.id], set))
eq_(sorted(ps.prod_to_topic[client_id_crm.id]),
['/cust/new', '/cust/update'])
self.assertTrue(isinstance(ps.prod_to_topic[client_id_billing.id],
set))
eq_(sorted(ps.prod_to_topic[client_id_billing.id]),
['/adsl/new', '/adsl/update'])
# Subscribe all the systems
sub_ctx_crm = SubCtx()
sub_ctx_crm.client_id = client_id_crm.id
sub_ctx_crm.topics = [topic_adsl_new.name, topic_adsl_update.name]
sub_ctx_billing = SubCtx()
sub_ctx_billing.client_id = client_id_billing.id
sub_ctx_billing.topics = [topic_cust_new.name, topic_cust_update.name]
sub_ctx_erp = SubCtx()
sub_ctx_erp.client_id = client_id_erp.id
sub_ctx_erp.topics = [topic_adsl_new.name, topic_adsl_update.name]
sub_key_crm = 'sub_key_crm'
sub_key_billing = 'sub_key_billing'
sub_key_erp = 'sub_key_erp'
received_sub_key_crm = ps.subscribe(sub_ctx_crm, sub_key_crm)
received_sub_key_billing = ps.subscribe(sub_ctx_billing,
sub_key_billing)
received_sub_key_erp = ps.subscribe(sub_ctx_erp, sub_key_erp)
eq_(sub_key_crm, received_sub_key_crm)
eq_(sub_key_billing, received_sub_key_billing)
eq_(sub_key_erp, received_sub_key_erp)
eq_(sorted(ps.sub_to_cons.items()), [('sub_key_billing', 'Billing'),
('sub_key_crm', 'CRM'),
('sub_key_erp', 'ERP')])
eq_(sorted(ps.cons_to_sub.items()), [('Billing', 'sub_key_billing'),
('CRM', 'sub_key_crm'),
('ERP', 'sub_key_erp')])
# CRM publishes Msg-CRM1 to /cust/new
pub_ctx_msg_crm1 = PubCtx()
pub_ctx_msg_crm1.client_id = client_id_crm.id
pub_ctx_msg_crm1.topic = topic_cust_new.name
pub_ctx_msg_crm1.msg = Message(msg_crm1_value,
mime_type='text/xml',
priority=1,
expiration=0.1)
msg_crm1_id = ps.publish(pub_ctx_msg_crm1).msg.msg_id
# CRM publishes Msg-CRM2 to /cust/new
pub_ctx_msg_crm2 = PubCtx()
pub_ctx_msg_crm2.client_id = client_id_crm.id
pub_ctx_msg_crm2.topic = topic_cust_update.name
pub_ctx_msg_crm2.msg = Message(msg_crm2_value,
mime_type='application/json',
priority=2,
expiration=0.2)
msg_crm2_id = ps.publish(pub_ctx_msg_crm2).msg.msg_id
# Billing publishes Msg-Billing1 to /adsl/new
pub_ctx_msg_billing1 = PubCtx()
pub_ctx_msg_billing1.client_id = client_id_billing.id
pub_ctx_msg_billing1.topic = topic_adsl_new.name
pub_ctx_msg_billing1.msg = Message(msg_billing1_value,
mime_type='application/soap+xml',
priority=3,
expiration=0.3)
msg_billing1_id = ps.publish(pub_ctx_msg_billing1).msg.msg_id
# Billing publishes Msg-Billing2 to /adsl/update
pub_ctx_msg_billing2 = PubCtx()
pub_ctx_msg_billing2.client_id = client_id_billing.id
pub_ctx_msg_billing2.topic = topic_adsl_update.name
# Nothing except payload and expiration set, defaults should be used
msg_billing2 = Message(msg_billing2_value, expiration=3600)
pub_ctx_msg_billing2.msg = msg_billing2
msg_billing2_id = ps.publish(pub_ctx_msg_billing2).msg.msg_id
keys = self.kvdb.keys('{}*'.format(self.key_prefix))
eq_(len(keys), 9)
expected_keys = [
ps.MSG_VALUES_KEY, ps.MSG_EXPIRE_AT_KEY, ps.LAST_PUB_TIME_KEY
]
for topic in topic_cust_new, topic_cust_update, topic_adsl_new, topic_adsl_update:
expected_keys.append(ps.MSG_IDS_PREFIX.format(topic.name))
for key in expected_keys:
self.assertIn(key, keys)
# Check values of messages published
self._check_msg_values_metadata(ps, msg_crm1_id, msg_crm2_id,
msg_billing1_id, msg_billing2_id, True)
# Now move the messages just published to each of the subscriber's queue.
# In a real environment this is done by a background job.
ps.move_to_target_queues()
# Now all the messages have been moved we can check if everything is in place
# ready for subscribers to get their messages.
keys = self.kvdb.keys('{}*'.format(self.key_prefix))
eq_(len(keys), 9)
self.assertIn(ps.UNACK_COUNTER_KEY, keys)
self.assertIn(ps.MSG_VALUES_KEY, keys)
for sub_key in (sub_key_crm, sub_key_billing, sub_key_erp):
key = ps.CONSUMER_MSG_IDS_PREFIX.format(sub_key)
self.assertIn(key, keys)
self._check_unack_counter(ps, msg_crm1_id, msg_crm2_id,
msg_billing1_id, msg_billing2_id, 1, 1, 2, 2)
# Check values of messages published are still there
self._check_msg_values_metadata(ps, msg_crm1_id, msg_crm2_id,
msg_billing1_id, msg_billing2_id, True)
# Check that each recipient has expected message IDs in its respective message queue
keys = self.kvdb.keys(ps.CONSUMER_MSG_IDS_PREFIX.format('*'))
eq_(len(keys), 3)
msg_ids_crm = self.kvdb.lrange(
ps.CONSUMER_MSG_IDS_PREFIX.format(sub_key_crm), 0, 100)
msg_ids_billing = self.kvdb.lrange(
ps.CONSUMER_MSG_IDS_PREFIX.format(sub_key_billing), 0, 100)
msg_ids_erp = self.kvdb.lrange(
ps.CONSUMER_MSG_IDS_PREFIX.format(sub_key_erp), 0, 100)
eq_(len(msg_ids_crm), 2)
eq_(len(msg_ids_billing), 2)
eq_(len(msg_ids_erp), 2)
self.assertIn(msg_billing1_id, msg_ids_crm)
self.assertIn(msg_billing2_id, msg_ids_crm)
self.assertIn(msg_billing1_id, msg_ids_erp)
self.assertIn(msg_billing2_id, msg_ids_erp)
self.assertIn(msg_crm1_id, msg_ids_billing)
self.assertIn(msg_crm1_id, msg_ids_billing)
# Now that the messages are in queues, let's fetch them
get_ctx_crm = GetCtx()
get_ctx_crm.sub_key = sub_key_crm
get_ctx_billing = GetCtx()
get_ctx_billing.sub_key = sub_key_billing
get_ctx_erp = GetCtx()
get_ctx_erp.sub_key = sub_key_erp
msgs_crm = sorted(list(ps.get(get_ctx_crm)), key=attrgetter('payload'))
msgs_billing = sorted(list(ps.get(get_ctx_billing)),
key=attrgetter('payload'))
msgs_erp = sorted(list(ps.get(get_ctx_erp)), key=attrgetter('payload'))
eq_(len(msgs_crm), 2)
eq_(len(msgs_billing), 2)
eq_(len(msgs_erp), 2)
self._assert_has_msg_metadata(msgs_crm, msg_billing1_id,
'application/soap+xml', 3, 0.3)
self._assert_has_msg_metadata(msgs_crm, msg_billing2_id, 'text/plain',
5, 3600)
self._assert_has_msg_metadata(msgs_billing, msg_crm1_id, 'text/xml', 1,
0.1)
self._assert_has_msg_metadata(msgs_billing, msg_crm2_id,
'application/json', 2, 0.2)
self._assert_has_msg_metadata(msgs_erp, msg_billing1_id,
'application/soap+xml', 3, 0.3)
self._assert_has_msg_metadata(msgs_erp, msg_billing2_id, 'text/plain',
5, 3600)
# Check in-flight status for each message got
keys = self.kvdb.keys(ps.CONSUMER_IN_FLIGHT_DATA_PREFIX.format('*'))
eq_(len(keys), 3)
now = arrow.utcnow()
self._check_in_flight(ps, now, sub_key_crm, sub_key_billing,
sub_key_erp, msg_crm1_id, msg_crm2_id,
msg_billing1_id, msg_billing2_id, True, True,
True, True, True, True)
# Messages should still be undelivered hence their unack counters are not touched at this point
self._check_unack_counter(ps, msg_crm1_id, msg_crm2_id,
msg_billing1_id, msg_billing2_id, 1, 1, 2, 2)
# ######################################################################################################################
# Messages are fetched, they can be confirmed or rejected now
# CRM
ack_ctx_crm = AckCtx()
ack_ctx_crm.sub_key = sub_key_crm
ack_ctx_crm.append(msg_billing1_id)
reject_ctx_crm = RejectCtx()
reject_ctx_crm.sub_key = sub_key_crm
reject_ctx_crm.append(msg_billing2_id)
ps.acknowledge_delete(ack_ctx_crm)
ps.reject(reject_ctx_crm)
# One in-flight less
self._check_in_flight(ps, now, sub_key_crm, sub_key_billing,
sub_key_erp, msg_crm1_id, msg_crm2_id,
msg_billing1_id, msg_billing2_id, False, False,
True, True, True, True)
# Rejections, as with reject_ctx_crm, don't change unack count
self._check_unack_counter(ps, msg_crm1_id, msg_crm2_id,
msg_billing1_id, msg_billing2_id, 1, 1, 1, 2)
# Billing
ack_ctx_billing = AckCtx()
ack_ctx_billing.sub_key = sub_key_billing
ack_ctx_billing.append(msg_crm1_id)
ack_ctx_billing.append(msg_crm2_id)
ps.acknowledge_delete(ack_ctx_billing)
# Two in-flight less
self._check_in_flight(ps, now, sub_key_crm, sub_key_billing,
sub_key_erp, msg_crm1_id, msg_crm2_id,
msg_billing1_id, msg_billing2_id, False, False,
False, False, True, True)
# Again, rejections, as with reject_ctx_crm, don't change unack count
self._check_unack_counter(ps, None, None, msg_billing1_id,
msg_billing2_id, 1, 1, 1, 2)
# ERP
reject_ctx_erp = RejectCtx()
reject_ctx_erp.sub_key = sub_key_erp
reject_ctx_erp.append(msg_billing1_id)
ack_ctx_erp = AckCtx()
ack_ctx_erp.sub_key = sub_key_erp
ack_ctx_erp.append(msg_billing2_id)
ps.reject(reject_ctx_erp)
ps.acknowledge_delete(ack_ctx_erp)
# Another in-flight less
self._check_in_flight(ps, now, sub_key_crm, sub_key_billing,
sub_key_erp, msg_crm1_id, msg_crm2_id,
msg_billing1_id, msg_billing2_id, False, False,
False, False, False, False)
# And again, rejections, as with reject_ctx_crm, don't change unack count
self._check_unack_counter(ps, None, None, msg_billing1_id,
msg_billing2_id, 1, 1, 1, 1)
# Sleep for a moment to make sure enough time passes for messages to expire
sleep(0.4)
# Deletes everything except for Msg-Billing2 which has a TTL of 3600
ps.delete_expired()
keys = self.kvdb.keys('{}*'.format(self.key_prefix))
eq_(len(keys), 8)
expected_keys = [
ps.MSG_VALUES_KEY, ps.MSG_EXPIRE_AT_KEY, ps.UNACK_COUNTER_KEY,
ps.CONSUMER_MSG_IDS_PREFIX.format(sub_key_crm)
]
for key in expected_keys:
self.assertTrue(key in keys,
'Key not found `{}` in `{}`'.format(key, keys))
# Check all the remaining keys - still concerning Msg-Billing2 only
# because this is the only message that wasn't confirmed nor expired.
expire_at = self.kvdb.hgetall(ps.MSG_EXPIRE_AT_KEY)
eq_(len(expire_at), 1)
eq_(expire_at[msg_billing2_id], msg_billing2.expire_at_utc.isoformat())
unack_counter = self.kvdb.hgetall(ps.UNACK_COUNTER_KEY)
eq_(len(unack_counter), 1)
eq_(unack_counter[msg_billing2_id], '1')
crm_messages = self.kvdb.lrange(
ps.CONSUMER_MSG_IDS_PREFIX.format(sub_key_crm), 0, 100)
eq_(len(crm_messages), 1)
eq_(crm_messages[0], msg_billing2_id)
msg_values = self.kvdb.hgetall(ps.MSG_METADATA_KEY)
eq_(len(msg_values), 2)
eq_(sorted(loads(msg_values[msg_billing2_id]).items()),
sorted(msg_billing2.to_dict().items()))