Example #1
0
    def test_reject_ctx_custom_attrs(self):
        sub_key = rand_string()
        msg_ids = rand_string(2)

        ctx = RejectCtx(sub_key, msg_ids)

        self.assertEquals(ctx.sub_key, sub_key)
        self.assertEquals(ctx.msg_ids, msg_ids)

        msg_id = rand_string()
        ctx.append(msg_id)

        self.assertEquals(ctx.msg_ids, msg_ids)
Example #2
0
    def test_reject_ctx_custom_attrs(self):
        sub_key = rand_string()
        msg_ids = rand_string(2)

        ctx = RejectCtx(sub_key, msg_ids)

        self.assertEquals(ctx.sub_key, sub_key)
        self.assertEquals(ctx.msg_ids, msg_ids)

        msg_id = rand_string()
        ctx.append(msg_id)

        self.assertEquals(ctx.msg_ids, msg_ids)
Example #3
0
 def test_reject_ctx_defaults(self):
     ctx = RejectCtx()
     self.assertEquals(ctx.sub_key, None)
     self.assertEquals(ctx.msg_ids, [])
    def test_full_path(self):
        """ Tests full sub/pub/ack/reject path with 4 topics and 3 clients. Doesn't test background tasks.
        """
        # Give up early if there is no connection to Redis at all
        if not self.has_redis:
            return

        """ What is tested.

        - 3 clients connect to pub/sub: CRM, Billing and ERP.
        - 4 topics are created: /cust/new, /cust/update, /adsl/new and /adsl/update

        - Subscriptions are:

          - CRM subs to /adsl/new
          - CRM subs to /adsl/update

          - Billing subs to /cust/new
          - Billing subs to /cust/update

          - ERP subs to /adsl/new
          - ERP subs to /adsl/update

        - Publications are:

          - CRM publishes Msg-CRM1 to /cust/new -------------- TTL of 0.1s
          - CRM publishes Msg-CRM2 to /cust/update ----------- TTL of 0.2s

          - Billing publishes Msg-Billing1 to /adsl/new ------ TTL of 0.3s
          - Billing publishes Msg-Billing2 to /adsl/update --- TTL of 3600s

          - (ERP doesn't publish anything)

        - Expected deliveries are:

          - Msg-CRM1 goes to Billing
          - Msg-CRM2 goes to Billing

          - Msg-Billing1 goes to CRM
          - Msg-Billing2 goes to CRM

          - Msg-Billing1 goes to ERP
          - Msg-Billing2 goes to ERP

        - Confirmations are:

          - CRM acks Msg-Billing1
          - CRM rejects Msg-Billing2

          - Billing acks Msg-CRM1
          - Billing acks Msg-CRM2

          - ERP rejects Msg-Billing1
          - ERP acks Msg-Billing2

        - Clean up tasks are

          - Msg-CRM1 is deleted because it's confirmed by its only recipient of Billing
          - Msg-CRM2 is deleted because it's confirmed by its only recipient of Billing

          - Msg-Billing1 is deleted because:

            - CRM confirms it
            - ERP rejects it but the message's TTL is 0.3s so it times out
              (In real world ERP would have possibly acknowledged it in say, 2s, but it would've been too late)

          - Msg-Billing2 is not deleted because:

            - ERP confirms it
            - CRM rejects it and the message's TTL is 3600s so it's still around when a clean up task runs

        """

        ps = RedisPubSub(self.kvdb, self.key_prefix)

        msg_billing1_value = '"msg_billing1"'
        msg_billing2_value = '"msg_billing2"'

        msg_crm1_value = '"msg_crm1"'
        msg_crm2_value = '"msg_crm2"'

        # Check all the Lua programs are loaded

        eq_(len(ps.lua_programs), 9)

        for attr in dir(ps):
            if attr.startswith('LUA'):
                value = getattr(ps, attr)
                self.assertTrue(value in ps.lua_programs, value)

        topic_cust_new = Topic('/cust/new')
        topic_cust_update = Topic('/cust/update')

        topic_adsl_new = Topic('/adsl/new')
        topic_adsl_update = Topic('/adsl/update')

        ps.add_topic(topic_cust_new)
        ps.add_topic(topic_cust_update)

        ps.add_topic(topic_adsl_new)
        ps.add_topic(topic_adsl_update)

        # Check all the topics are cached locally

        eq_(len(ps.topics), 4)

        for topic in(topic_cust_new, topic_cust_update, topic_adsl_new, topic_adsl_update):
            eq_(ps.topics[topic.name], topic)

        client_id_crm = Consumer('CRM', 'CRM', sub_key='sub_key_crm')
        client_id_billing = Consumer('Billing', 'Billing', sub_key='sub_key_billing')
        client_id_erp = Consumer('ERP', 'ERP', sub_key='sub_key_crm')

        ps.add_producer(client_id_crm, topic_cust_new)
        ps.add_producer(client_id_crm, topic_cust_update)

        ps.add_producer(client_id_billing, topic_adsl_new)
        ps.add_producer(client_id_billing, topic_adsl_update)

        ps.add_consumer(client_id_crm, topic_adsl_new)
        ps.add_consumer(client_id_crm, topic_adsl_update)

        ps.add_consumer(client_id_billing, topic_cust_new)
        ps.add_consumer(client_id_billing, topic_cust_update)

        ps.add_consumer(client_id_erp, topic_adsl_new)
        ps.add_consumer(client_id_erp, topic_adsl_update)

        # Check producers have been registered for topics

        eq_(len(ps.prod_to_topic), 2)

        self.assertTrue(client_id_crm.id in ps.prod_to_topic)
        self.assertTrue(client_id_billing.id in ps.prod_to_topic)
        self.assertTrue(client_id_erp.id not in ps.prod_to_topic)

        self.assertTrue(isinstance(ps.prod_to_topic[client_id_crm.id], set))
        eq_(sorted(ps.prod_to_topic[client_id_crm.id]), ['/cust/new', '/cust/update'])

        self.assertTrue(isinstance(ps.prod_to_topic[client_id_billing.id], set))
        eq_(sorted(ps.prod_to_topic[client_id_billing.id]), ['/adsl/new', '/adsl/update'])

        # Subscribe all the systems

        sub_ctx_crm = SubCtx()
        sub_ctx_crm.client_id = client_id_crm.id
        sub_ctx_crm.topics = [topic_adsl_new.name, topic_adsl_update.name]

        sub_ctx_billing = SubCtx()
        sub_ctx_billing.client_id = client_id_billing.id
        sub_ctx_billing.topics = [topic_cust_new.name, topic_cust_update.name]

        sub_ctx_erp = SubCtx()
        sub_ctx_erp.client_id = client_id_erp.id
        sub_ctx_erp.topics = [topic_adsl_new.name, topic_adsl_update.name]

        sub_key_crm = 'sub_key_crm'
        sub_key_billing = 'sub_key_billing'
        sub_key_erp = 'sub_key_erp'

        received_sub_key_crm = ps.subscribe(sub_ctx_crm, sub_key_crm)
        received_sub_key_billing = ps.subscribe(sub_ctx_billing, sub_key_billing)
        received_sub_key_erp = ps.subscribe(sub_ctx_erp, sub_key_erp)

        eq_(sub_key_crm, received_sub_key_crm)
        eq_(sub_key_billing, received_sub_key_billing)
        eq_(sub_key_erp, received_sub_key_erp)

        eq_(sorted(ps.sub_to_cons.items()), [('sub_key_billing', 'Billing'), ('sub_key_crm', 'CRM'), ('sub_key_erp', 'ERP')])
        eq_(sorted(ps.cons_to_sub.items()), [('Billing', 'sub_key_billing'), ('CRM', 'sub_key_crm'), ('ERP', 'sub_key_erp')])

        # CRM publishes Msg-CRM1 to /cust/new
        pub_ctx_msg_crm1 = PubCtx()
        pub_ctx_msg_crm1.client_id = client_id_crm.id
        pub_ctx_msg_crm1.topic = topic_cust_new.name
        pub_ctx_msg_crm1.msg = Message(msg_crm1_value, mime_type='text/xml', priority=1, expiration=0.1)

        msg_crm1_id = ps.publish(pub_ctx_msg_crm1).msg.msg_id

        # CRM publishes Msg-CRM2 to /cust/new
        pub_ctx_msg_crm2 = PubCtx()
        pub_ctx_msg_crm2.client_id = client_id_crm.id
        pub_ctx_msg_crm2.topic = topic_cust_update.name
        pub_ctx_msg_crm2.msg = Message(msg_crm2_value, mime_type='application/json', priority=2, expiration=0.2)

        msg_crm2_id = ps.publish(pub_ctx_msg_crm2).msg.msg_id

        # Billing publishes Msg-Billing1 to /adsl/new
        pub_ctx_msg_billing1 = PubCtx()
        pub_ctx_msg_billing1.client_id = client_id_billing.id
        pub_ctx_msg_billing1.topic = topic_adsl_new.name
        pub_ctx_msg_billing1.msg = Message(msg_billing1_value, mime_type='application/soap+xml', priority=3, expiration=0.3)

        msg_billing1_id = ps.publish(pub_ctx_msg_billing1).msg.msg_id

        # Billing publishes Msg-Billing2 to /adsl/update
        pub_ctx_msg_billing2 = PubCtx()
        pub_ctx_msg_billing2.client_id = client_id_billing.id
        pub_ctx_msg_billing2.topic = topic_adsl_update.name

        # Nothing except payload and expiration set, defaults should be used
        msg_billing2 = Message(msg_billing2_value, expiration=3600)
        pub_ctx_msg_billing2.msg = msg_billing2

        msg_billing2_id = ps.publish(pub_ctx_msg_billing2).msg.msg_id

        keys = self.kvdb.keys('{}*'.format(self.key_prefix))
        eq_(len(keys), 9)

        expected_keys = [ps.MSG_VALUES_KEY, ps.MSG_EXPIRE_AT_KEY, ps.LAST_PUB_TIME_KEY]
        for topic in topic_cust_new, topic_cust_update, topic_adsl_new, topic_adsl_update:
            expected_keys.append(ps.MSG_IDS_PREFIX.format(topic.name))

        for key in expected_keys:
            self.assertIn(key, keys)

        # Check values of messages published
        self._check_msg_values_metadata(ps, msg_crm1_id, msg_crm2_id, msg_billing1_id, msg_billing2_id, True)

        # Now move the messages just published to each of the subscriber's queue.
        # In a real environment this is done by a background job.
        ps.move_to_target_queues()

        # Now all the messages have been moved we can check if everything is in place
        # ready for subscribers to get their messages.

        keys = self.kvdb.keys('{}*'.format(self.key_prefix))
        eq_(len(keys), 9)

        self.assertIn(ps.UNACK_COUNTER_KEY, keys)
        self.assertIn(ps.MSG_VALUES_KEY, keys)

        for sub_key in(sub_key_crm, sub_key_billing, sub_key_erp):
            key = ps.CONSUMER_MSG_IDS_PREFIX.format(sub_key)
            self.assertIn(key, keys)

        self._check_unack_counter(ps, msg_crm1_id, msg_crm2_id, msg_billing1_id, msg_billing2_id, 1, 1, 2, 2)

        # Check values of messages published are still there
        self._check_msg_values_metadata(ps, msg_crm1_id, msg_crm2_id, msg_billing1_id, msg_billing2_id, True)

        # Check that each recipient has expected message IDs in its respective message queue
        keys = self.kvdb.keys(ps.CONSUMER_MSG_IDS_PREFIX.format('*'))
        eq_(len(keys), 3)

        msg_ids_crm = self.kvdb.lrange(ps.CONSUMER_MSG_IDS_PREFIX.format(sub_key_crm), 0, 100)
        msg_ids_billing = self.kvdb.lrange(ps.CONSUMER_MSG_IDS_PREFIX.format(sub_key_billing), 0, 100)
        msg_ids_erp = self.kvdb.lrange(ps.CONSUMER_MSG_IDS_PREFIX.format(sub_key_erp), 0, 100)

        eq_(len(msg_ids_crm), 2)
        eq_(len(msg_ids_billing), 2)
        eq_(len(msg_ids_erp), 2)

        self.assertIn(msg_billing1_id, msg_ids_crm)
        self.assertIn(msg_billing2_id, msg_ids_crm)

        self.assertIn(msg_billing1_id, msg_ids_erp)
        self.assertIn(msg_billing2_id, msg_ids_erp)

        self.assertIn(msg_crm1_id, msg_ids_billing)
        self.assertIn(msg_crm1_id, msg_ids_billing)

        # Now that the messages are in queues, let's fetch them

        get_ctx_crm = GetCtx()
        get_ctx_crm.sub_key = sub_key_crm

        get_ctx_billing = GetCtx()
        get_ctx_billing.sub_key = sub_key_billing

        get_ctx_erp = GetCtx()
        get_ctx_erp.sub_key = sub_key_erp

        msgs_crm = sorted(list(ps.get(get_ctx_crm)), key=attrgetter('payload'))
        msgs_billing = sorted(list(ps.get(get_ctx_billing)), key=attrgetter('payload'))
        msgs_erp = sorted(list(ps.get(get_ctx_erp)), key=attrgetter('payload'))

        eq_(len(msgs_crm), 2)
        eq_(len(msgs_billing), 2)
        eq_(len(msgs_erp), 2)

        self._assert_has_msg_metadata(msgs_crm, msg_billing1_id, 'application/soap+xml', 3, 0.3)
        self._assert_has_msg_metadata(msgs_crm, msg_billing2_id, 'text/plain', 5, 3600)

        self._assert_has_msg_metadata(msgs_billing, msg_crm1_id, 'text/xml', 1, 0.1)
        self._assert_has_msg_metadata(msgs_billing, msg_crm2_id, 'application/json', 2, 0.2)

        self._assert_has_msg_metadata(msgs_erp, msg_billing1_id, 'application/soap+xml', 3, 0.3)
        self._assert_has_msg_metadata(msgs_erp, msg_billing2_id, 'text/plain', 5, 3600)

        # Check in-flight status for each message got
        keys = self.kvdb.keys(ps.CONSUMER_IN_FLIGHT_DATA_PREFIX.format('*'))
        eq_(len(keys), 3)

        now = arrow.utcnow()

        self._check_in_flight(ps, now, sub_key_crm, sub_key_billing, sub_key_erp, msg_crm1_id, msg_crm2_id,
                msg_billing1_id, msg_billing2_id, True, True, True, True, True, True)

        # Messages should still be undelivered hence their unack counters are not touched at this point
        self._check_unack_counter(ps, msg_crm1_id, msg_crm2_id, msg_billing1_id, msg_billing2_id, 1, 1, 2, 2)

# ######################################################################################################################

        # Messages are fetched, they can be confirmed or rejected now

        # CRM
        ack_ctx_crm = AckCtx()
        ack_ctx_crm.sub_key = sub_key_crm
        ack_ctx_crm.append(msg_billing1_id)

        reject_ctx_crm = RejectCtx()
        reject_ctx_crm.sub_key = sub_key_crm
        reject_ctx_crm.append(msg_billing2_id)

        ps.acknowledge_delete(ack_ctx_crm)
        ps.reject(reject_ctx_crm)

        # One in-flight less
        self._check_in_flight(ps, now, sub_key_crm, sub_key_billing, sub_key_erp, msg_crm1_id, msg_crm2_id,
                msg_billing1_id, msg_billing2_id, False, False, True, True, True, True)

        # Rejections, as with reject_ctx_crm, don't change unack count
        self._check_unack_counter(ps, msg_crm1_id, msg_crm2_id, msg_billing1_id, msg_billing2_id, 1, 1, 1, 2)

        # Billing
        ack_ctx_billing = AckCtx()
        ack_ctx_billing.sub_key = sub_key_billing
        ack_ctx_billing.append(msg_crm1_id)
        ack_ctx_billing.append(msg_crm2_id)

        ps.acknowledge_delete(ack_ctx_billing)

        # Two in-flight less
        self._check_in_flight(ps, now, sub_key_crm, sub_key_billing, sub_key_erp, msg_crm1_id, msg_crm2_id,
                msg_billing1_id, msg_billing2_id, False, False, False, False, True, True)

        # Again, rejections, as with reject_ctx_crm, don't change unack count
        self._check_unack_counter(ps, None, None, msg_billing1_id, msg_billing2_id, 1, 1, 1, 2)

        # ERP
        reject_ctx_erp = RejectCtx()
        reject_ctx_erp.sub_key = sub_key_erp
        reject_ctx_erp.append(msg_billing1_id)

        ack_ctx_erp = AckCtx()
        ack_ctx_erp.sub_key = sub_key_erp
        ack_ctx_erp.append(msg_billing2_id)

        ps.reject(reject_ctx_erp)
        ps.acknowledge_delete(ack_ctx_erp)

        # Another in-flight less
        self._check_in_flight(ps, now, sub_key_crm, sub_key_billing, sub_key_erp, msg_crm1_id, msg_crm2_id,
                msg_billing1_id, msg_billing2_id, False, False, False, False, False, False)

        # And again, rejections, as with reject_ctx_crm, don't change unack count
        self._check_unack_counter(ps, None, None, msg_billing1_id, msg_billing2_id, 1, 1, 1, 1)

        # Sleep for a moment to make sure enough time passes for messages to expire
        sleep(0.4)

        # Deletes everything except for Msg-Billing2 which has a TTL of 3600
        ps.delete_expired()

        keys = self.kvdb.keys('{}*'.format(self.key_prefix))
        eq_(len(keys), 8)

        expected_keys = [ps.MSG_VALUES_KEY, ps.MSG_EXPIRE_AT_KEY, ps.UNACK_COUNTER_KEY,
                         ps.CONSUMER_MSG_IDS_PREFIX.format(sub_key_crm)]
        for key in expected_keys:
            self.assertTrue(key in keys, 'Key not found `{}` in `{}`'.format(key, keys))

        # Check all the remaining keys - still concerning Msg-Billing2 only
        # because this is the only message that wasn't confirmed nor expired.

        expire_at = self.kvdb.hgetall(ps.MSG_EXPIRE_AT_KEY)
        eq_(len(expire_at), 1)
        eq_(expire_at[msg_billing2_id], msg_billing2.expire_at_utc.isoformat())

        unack_counter = self.kvdb.hgetall(ps.UNACK_COUNTER_KEY)
        eq_(len(unack_counter), 1)
        eq_(unack_counter[msg_billing2_id], '1')

        crm_messages = self.kvdb.lrange(ps.CONSUMER_MSG_IDS_PREFIX.format(sub_key_crm), 0, 100)
        eq_(len(crm_messages), 1)
        eq_(crm_messages[0],msg_billing2_id)

        msg_values = self.kvdb.hgetall(ps.MSG_METADATA_KEY)
        eq_(len(msg_values), 2)
        eq_(sorted(loads(msg_values[msg_billing2_id]).items()), sorted(msg_billing2.to_dict().items()))
Example #5
0
    def test_full_path(self):
        """ Tests full sub/pub/ack/reject path with 4 topics and 3 clients. Doesn't test background tasks.
        """
        # Give up early if there is no connection to Redis at all
        if not self.has_redis:
            return
        """ What is tested.

        - 3 clients connect to pub/sub: CRM, Billing and ERP.
        - 4 topics are created: /cust/new, /cust/update, /adsl/new and /adsl/update

        - Subscriptions are:

          - CRM subs to /adsl/new
          - CRM subs to /adsl/update

          - Billing subs to /cust/new
          - Billing subs to /cust/update

          - ERP subs to /adsl/new
          - ERP subs to /adsl/update

        - Publications are:

          - CRM publishes Msg-CRM1 to /cust/new -------------- TTL of 0.1s
          - CRM publishes Msg-CRM2 to /cust/update ----------- TTL of 0.2s

          - Billing publishes Msg-Billing1 to /adsl/new ------ TTL of 0.3s
          - Billing publishes Msg-Billing2 to /adsl/update --- TTL of 3600s

          - (ERP doesn't publish anything)

        - Expected deliveries are:

          - Msg-CRM1 goes to Billing
          - Msg-CRM2 goes to Billing

          - Msg-Billing1 goes to CRM
          - Msg-Billing2 goes to CRM

          - Msg-Billing1 goes to ERP
          - Msg-Billing2 goes to ERP

        - Confirmations are:

          - CRM acks Msg-Billing1
          - CRM rejects Msg-Billing2

          - Billing acks Msg-CRM1
          - Billing acks Msg-CRM2

          - ERP rejects Msg-Billing1
          - ERP acks Msg-Billing2

        - Clean up tasks are

          - Msg-CRM1 is deleted because it's confirmed by its only recipient of Billing
          - Msg-CRM2 is deleted because it's confirmed by its only recipient of Billing

          - Msg-Billing1 is deleted because:

            - CRM confirms it
            - ERP rejects it but the message's TTL is 0.3s so it times out
              (In real world ERP would have possibly acknowledged it in say, 2s, but it would've been too late)

          - Msg-Billing2 is not deleted because:

            - ERP confirms it
            - CRM rejects it and the message's TTL is 3600s so it's still around when a clean up task runs

        """

        ps = RedisPubSub(self.kvdb, self.key_prefix)

        msg_billing1_value = '"msg_billing1"'
        msg_billing2_value = '"msg_billing2"'

        msg_crm1_value = '"msg_crm1"'
        msg_crm2_value = '"msg_crm2"'

        # Check all the Lua programs are loaded

        eq_(len(ps.lua_programs), 9)

        for attr in dir(ps):
            if attr.startswith('LUA'):
                value = getattr(ps, attr)
                self.assertTrue(value in ps.lua_programs, value)

        topic_cust_new = Topic('/cust/new')
        topic_cust_update = Topic('/cust/update')

        topic_adsl_new = Topic('/adsl/new')
        topic_adsl_update = Topic('/adsl/update')

        ps.add_topic(topic_cust_new)
        ps.add_topic(topic_cust_update)

        ps.add_topic(topic_adsl_new)
        ps.add_topic(topic_adsl_update)

        # Check all the topics are cached locally

        eq_(len(ps.topics), 4)

        for topic in (topic_cust_new, topic_cust_update, topic_adsl_new,
                      topic_adsl_update):
            eq_(ps.topics[topic.name], topic)

        client_id_crm = Consumer('CRM', 'CRM', sub_key='sub_key_crm')
        client_id_billing = Consumer('Billing',
                                     'Billing',
                                     sub_key='sub_key_billing')
        client_id_erp = Consumer('ERP', 'ERP', sub_key='sub_key_crm')

        ps.add_producer(client_id_crm, topic_cust_new)
        ps.add_producer(client_id_crm, topic_cust_update)

        ps.add_producer(client_id_billing, topic_adsl_new)
        ps.add_producer(client_id_billing, topic_adsl_update)

        ps.add_consumer(client_id_crm, topic_adsl_new)
        ps.add_consumer(client_id_crm, topic_adsl_update)

        ps.add_consumer(client_id_billing, topic_cust_new)
        ps.add_consumer(client_id_billing, topic_cust_update)

        ps.add_consumer(client_id_erp, topic_adsl_new)
        ps.add_consumer(client_id_erp, topic_adsl_update)

        # Check producers have been registered for topics

        eq_(len(ps.prod_to_topic), 2)

        self.assertTrue(client_id_crm.id in ps.prod_to_topic)
        self.assertTrue(client_id_billing.id in ps.prod_to_topic)
        self.assertTrue(client_id_erp.id not in ps.prod_to_topic)

        self.assertTrue(isinstance(ps.prod_to_topic[client_id_crm.id], set))
        eq_(sorted(ps.prod_to_topic[client_id_crm.id]),
            ['/cust/new', '/cust/update'])

        self.assertTrue(isinstance(ps.prod_to_topic[client_id_billing.id],
                                   set))
        eq_(sorted(ps.prod_to_topic[client_id_billing.id]),
            ['/adsl/new', '/adsl/update'])

        # Subscribe all the systems

        sub_ctx_crm = SubCtx()
        sub_ctx_crm.client_id = client_id_crm.id
        sub_ctx_crm.topics = [topic_adsl_new.name, topic_adsl_update.name]

        sub_ctx_billing = SubCtx()
        sub_ctx_billing.client_id = client_id_billing.id
        sub_ctx_billing.topics = [topic_cust_new.name, topic_cust_update.name]

        sub_ctx_erp = SubCtx()
        sub_ctx_erp.client_id = client_id_erp.id
        sub_ctx_erp.topics = [topic_adsl_new.name, topic_adsl_update.name]

        sub_key_crm = 'sub_key_crm'
        sub_key_billing = 'sub_key_billing'
        sub_key_erp = 'sub_key_erp'

        received_sub_key_crm = ps.subscribe(sub_ctx_crm, sub_key_crm)
        received_sub_key_billing = ps.subscribe(sub_ctx_billing,
                                                sub_key_billing)
        received_sub_key_erp = ps.subscribe(sub_ctx_erp, sub_key_erp)

        eq_(sub_key_crm, received_sub_key_crm)
        eq_(sub_key_billing, received_sub_key_billing)
        eq_(sub_key_erp, received_sub_key_erp)

        eq_(sorted(ps.sub_to_cons.items()), [('sub_key_billing', 'Billing'),
                                             ('sub_key_crm', 'CRM'),
                                             ('sub_key_erp', 'ERP')])
        eq_(sorted(ps.cons_to_sub.items()), [('Billing', 'sub_key_billing'),
                                             ('CRM', 'sub_key_crm'),
                                             ('ERP', 'sub_key_erp')])

        # CRM publishes Msg-CRM1 to /cust/new
        pub_ctx_msg_crm1 = PubCtx()
        pub_ctx_msg_crm1.client_id = client_id_crm.id
        pub_ctx_msg_crm1.topic = topic_cust_new.name
        pub_ctx_msg_crm1.msg = Message(msg_crm1_value,
                                       mime_type='text/xml',
                                       priority=1,
                                       expiration=0.1)

        msg_crm1_id = ps.publish(pub_ctx_msg_crm1).msg.msg_id

        # CRM publishes Msg-CRM2 to /cust/new
        pub_ctx_msg_crm2 = PubCtx()
        pub_ctx_msg_crm2.client_id = client_id_crm.id
        pub_ctx_msg_crm2.topic = topic_cust_update.name
        pub_ctx_msg_crm2.msg = Message(msg_crm2_value,
                                       mime_type='application/json',
                                       priority=2,
                                       expiration=0.2)

        msg_crm2_id = ps.publish(pub_ctx_msg_crm2).msg.msg_id

        # Billing publishes Msg-Billing1 to /adsl/new
        pub_ctx_msg_billing1 = PubCtx()
        pub_ctx_msg_billing1.client_id = client_id_billing.id
        pub_ctx_msg_billing1.topic = topic_adsl_new.name
        pub_ctx_msg_billing1.msg = Message(msg_billing1_value,
                                           mime_type='application/soap+xml',
                                           priority=3,
                                           expiration=0.3)

        msg_billing1_id = ps.publish(pub_ctx_msg_billing1).msg.msg_id

        # Billing publishes Msg-Billing2 to /adsl/update
        pub_ctx_msg_billing2 = PubCtx()
        pub_ctx_msg_billing2.client_id = client_id_billing.id
        pub_ctx_msg_billing2.topic = topic_adsl_update.name

        # Nothing except payload and expiration set, defaults should be used
        msg_billing2 = Message(msg_billing2_value, expiration=3600)
        pub_ctx_msg_billing2.msg = msg_billing2

        msg_billing2_id = ps.publish(pub_ctx_msg_billing2).msg.msg_id

        keys = self.kvdb.keys('{}*'.format(self.key_prefix))
        eq_(len(keys), 9)

        expected_keys = [
            ps.MSG_VALUES_KEY, ps.MSG_EXPIRE_AT_KEY, ps.LAST_PUB_TIME_KEY
        ]
        for topic in topic_cust_new, topic_cust_update, topic_adsl_new, topic_adsl_update:
            expected_keys.append(ps.MSG_IDS_PREFIX.format(topic.name))

        for key in expected_keys:
            self.assertIn(key, keys)

        # Check values of messages published
        self._check_msg_values_metadata(ps, msg_crm1_id, msg_crm2_id,
                                        msg_billing1_id, msg_billing2_id, True)

        # Now move the messages just published to each of the subscriber's queue.
        # In a real environment this is done by a background job.
        ps.move_to_target_queues()

        # Now all the messages have been moved we can check if everything is in place
        # ready for subscribers to get their messages.

        keys = self.kvdb.keys('{}*'.format(self.key_prefix))
        eq_(len(keys), 9)

        self.assertIn(ps.UNACK_COUNTER_KEY, keys)
        self.assertIn(ps.MSG_VALUES_KEY, keys)

        for sub_key in (sub_key_crm, sub_key_billing, sub_key_erp):
            key = ps.CONSUMER_MSG_IDS_PREFIX.format(sub_key)
            self.assertIn(key, keys)

        self._check_unack_counter(ps, msg_crm1_id, msg_crm2_id,
                                  msg_billing1_id, msg_billing2_id, 1, 1, 2, 2)

        # Check values of messages published are still there
        self._check_msg_values_metadata(ps, msg_crm1_id, msg_crm2_id,
                                        msg_billing1_id, msg_billing2_id, True)

        # Check that each recipient has expected message IDs in its respective message queue
        keys = self.kvdb.keys(ps.CONSUMER_MSG_IDS_PREFIX.format('*'))
        eq_(len(keys), 3)

        msg_ids_crm = self.kvdb.lrange(
            ps.CONSUMER_MSG_IDS_PREFIX.format(sub_key_crm), 0, 100)
        msg_ids_billing = self.kvdb.lrange(
            ps.CONSUMER_MSG_IDS_PREFIX.format(sub_key_billing), 0, 100)
        msg_ids_erp = self.kvdb.lrange(
            ps.CONSUMER_MSG_IDS_PREFIX.format(sub_key_erp), 0, 100)

        eq_(len(msg_ids_crm), 2)
        eq_(len(msg_ids_billing), 2)
        eq_(len(msg_ids_erp), 2)

        self.assertIn(msg_billing1_id, msg_ids_crm)
        self.assertIn(msg_billing2_id, msg_ids_crm)

        self.assertIn(msg_billing1_id, msg_ids_erp)
        self.assertIn(msg_billing2_id, msg_ids_erp)

        self.assertIn(msg_crm1_id, msg_ids_billing)
        self.assertIn(msg_crm1_id, msg_ids_billing)

        # Now that the messages are in queues, let's fetch them

        get_ctx_crm = GetCtx()
        get_ctx_crm.sub_key = sub_key_crm

        get_ctx_billing = GetCtx()
        get_ctx_billing.sub_key = sub_key_billing

        get_ctx_erp = GetCtx()
        get_ctx_erp.sub_key = sub_key_erp

        msgs_crm = sorted(list(ps.get(get_ctx_crm)), key=attrgetter('payload'))
        msgs_billing = sorted(list(ps.get(get_ctx_billing)),
                              key=attrgetter('payload'))
        msgs_erp = sorted(list(ps.get(get_ctx_erp)), key=attrgetter('payload'))

        eq_(len(msgs_crm), 2)
        eq_(len(msgs_billing), 2)
        eq_(len(msgs_erp), 2)

        self._assert_has_msg_metadata(msgs_crm, msg_billing1_id,
                                      'application/soap+xml', 3, 0.3)
        self._assert_has_msg_metadata(msgs_crm, msg_billing2_id, 'text/plain',
                                      5, 3600)

        self._assert_has_msg_metadata(msgs_billing, msg_crm1_id, 'text/xml', 1,
                                      0.1)
        self._assert_has_msg_metadata(msgs_billing, msg_crm2_id,
                                      'application/json', 2, 0.2)

        self._assert_has_msg_metadata(msgs_erp, msg_billing1_id,
                                      'application/soap+xml', 3, 0.3)
        self._assert_has_msg_metadata(msgs_erp, msg_billing2_id, 'text/plain',
                                      5, 3600)

        # Check in-flight status for each message got
        keys = self.kvdb.keys(ps.CONSUMER_IN_FLIGHT_DATA_PREFIX.format('*'))
        eq_(len(keys), 3)

        now = arrow.utcnow()

        self._check_in_flight(ps, now, sub_key_crm, sub_key_billing,
                              sub_key_erp, msg_crm1_id, msg_crm2_id,
                              msg_billing1_id, msg_billing2_id, True, True,
                              True, True, True, True)

        # Messages should still be undelivered hence their unack counters are not touched at this point
        self._check_unack_counter(ps, msg_crm1_id, msg_crm2_id,
                                  msg_billing1_id, msg_billing2_id, 1, 1, 2, 2)

        # ######################################################################################################################

        # Messages are fetched, they can be confirmed or rejected now

        # CRM
        ack_ctx_crm = AckCtx()
        ack_ctx_crm.sub_key = sub_key_crm
        ack_ctx_crm.append(msg_billing1_id)

        reject_ctx_crm = RejectCtx()
        reject_ctx_crm.sub_key = sub_key_crm
        reject_ctx_crm.append(msg_billing2_id)

        ps.acknowledge_delete(ack_ctx_crm)
        ps.reject(reject_ctx_crm)

        # One in-flight less
        self._check_in_flight(ps, now, sub_key_crm, sub_key_billing,
                              sub_key_erp, msg_crm1_id, msg_crm2_id,
                              msg_billing1_id, msg_billing2_id, False, False,
                              True, True, True, True)

        # Rejections, as with reject_ctx_crm, don't change unack count
        self._check_unack_counter(ps, msg_crm1_id, msg_crm2_id,
                                  msg_billing1_id, msg_billing2_id, 1, 1, 1, 2)

        # Billing
        ack_ctx_billing = AckCtx()
        ack_ctx_billing.sub_key = sub_key_billing
        ack_ctx_billing.append(msg_crm1_id)
        ack_ctx_billing.append(msg_crm2_id)

        ps.acknowledge_delete(ack_ctx_billing)

        # Two in-flight less
        self._check_in_flight(ps, now, sub_key_crm, sub_key_billing,
                              sub_key_erp, msg_crm1_id, msg_crm2_id,
                              msg_billing1_id, msg_billing2_id, False, False,
                              False, False, True, True)

        # Again, rejections, as with reject_ctx_crm, don't change unack count
        self._check_unack_counter(ps, None, None, msg_billing1_id,
                                  msg_billing2_id, 1, 1, 1, 2)

        # ERP
        reject_ctx_erp = RejectCtx()
        reject_ctx_erp.sub_key = sub_key_erp
        reject_ctx_erp.append(msg_billing1_id)

        ack_ctx_erp = AckCtx()
        ack_ctx_erp.sub_key = sub_key_erp
        ack_ctx_erp.append(msg_billing2_id)

        ps.reject(reject_ctx_erp)
        ps.acknowledge_delete(ack_ctx_erp)

        # Another in-flight less
        self._check_in_flight(ps, now, sub_key_crm, sub_key_billing,
                              sub_key_erp, msg_crm1_id, msg_crm2_id,
                              msg_billing1_id, msg_billing2_id, False, False,
                              False, False, False, False)

        # And again, rejections, as with reject_ctx_crm, don't change unack count
        self._check_unack_counter(ps, None, None, msg_billing1_id,
                                  msg_billing2_id, 1, 1, 1, 1)

        # Sleep for a moment to make sure enough time passes for messages to expire
        sleep(0.4)

        # Deletes everything except for Msg-Billing2 which has a TTL of 3600
        ps.delete_expired()

        keys = self.kvdb.keys('{}*'.format(self.key_prefix))
        eq_(len(keys), 8)

        expected_keys = [
            ps.MSG_VALUES_KEY, ps.MSG_EXPIRE_AT_KEY, ps.UNACK_COUNTER_KEY,
            ps.CONSUMER_MSG_IDS_PREFIX.format(sub_key_crm)
        ]
        for key in expected_keys:
            self.assertTrue(key in keys,
                            'Key not found `{}` in `{}`'.format(key, keys))

        # Check all the remaining keys - still concerning Msg-Billing2 only
        # because this is the only message that wasn't confirmed nor expired.

        expire_at = self.kvdb.hgetall(ps.MSG_EXPIRE_AT_KEY)
        eq_(len(expire_at), 1)
        eq_(expire_at[msg_billing2_id], msg_billing2.expire_at_utc.isoformat())

        unack_counter = self.kvdb.hgetall(ps.UNACK_COUNTER_KEY)
        eq_(len(unack_counter), 1)
        eq_(unack_counter[msg_billing2_id], '1')

        crm_messages = self.kvdb.lrange(
            ps.CONSUMER_MSG_IDS_PREFIX.format(sub_key_crm), 0, 100)
        eq_(len(crm_messages), 1)
        eq_(crm_messages[0], msg_billing2_id)

        msg_values = self.kvdb.hgetall(ps.MSG_METADATA_KEY)
        eq_(len(msg_values), 2)
        eq_(sorted(loads(msg_values[msg_billing2_id]).items()),
            sorted(msg_billing2.to_dict().items()))