Esempio n. 1
0
    def test_sub_key_generation(self):
        """ Checks whether a sub_key is generated if none is provided on input during subscribing.
        """
        sub_ctx = SubCtx()
        sub_ctx.client_id = '111'
        sub_ctx.topics = ['aaa']

        ps = RedisPubSub(self.kvdb)
        sub_key = ps.subscribe(sub_ctx)

        # Does it looks like a CID?
        eq_(len(sub_key), 24)
    def test_sub_key_generation(self):
        """ Checks whether a sub_key is generated if none is provided on input during subscribing.
        """
        sub_ctx = SubCtx()
        sub_ctx.client_id = '111'
        sub_ctx.topics = ['aaa']

        ps = RedisPubSub(self.kvdb)
        sub_key = ps.subscribe(sub_ctx)

        # Does it looks like a CID?
        eq_(len(sub_key), 24)
Esempio n. 3
0
    def test_delete_from_topic_no_subscribers(self):
        """ Tests deleting of messages from a topic without subscribers.
        """
        ps = RedisPubSub(self.kvdb, self.key_prefix)

        msg_value = '"msg_value"'

        topic = Topic('/test/delete')
        ps.add_topic(topic)

        producer = Client('Producer', 'producer')
        ps.add_producer(producer, topic)

        pub_ctx = PubCtx()
        pub_ctx.client_id = producer.id
        pub_ctx.topic = topic.name
        pub_ctx.msg = Message(msg_value)

        msg_id = ps.publish(pub_ctx).msg.msg_id

        # We've got a msg_id now and we know this is a topic with no subsribers. In that case, deleting a messages should
        # also delete all the metadata related to it along with its payload.

        # First, let's confirm the messages is actually there

        result = self.kvdb.zrange(ps.MSG_IDS_PREFIX.format(topic.name), 0, -1)
        eq_(result, [msg_id])

        result = self.kvdb.hgetall(ps.MSG_VALUES_KEY)
        eq_(len(result), 1)
        self.assertTrue(msg_id in result)

        result = self.kvdb.hgetall(ps.MSG_METADATA_KEY)
        eq_(len(result), 1)
        self.assertTrue(msg_id in result)

        result = self.kvdb.hgetall(ps.MSG_EXPIRE_AT_KEY)
        eq_(len(result), 1)
        self.assertTrue(msg_id in result)

        # Ok, now delete the message and confirm it's not in Redis anymore

        ps.delete_from_topic(topic.name, msg_id)

        result = self.kvdb.zrange(ps.MSG_IDS_PREFIX.format(topic.name), 0, -1)
        eq_(result, [])

        result = self.kvdb.hgetall(ps.MSG_VALUES_KEY)
        eq_({}, result)

        result = self.kvdb.hgetall(ps.MSG_METADATA_KEY)
        eq_({}, result)

        result = self.kvdb.hgetall(ps.MSG_EXPIRE_AT_KEY)
        eq_({}, result)
    def test_delete_from_topic_no_subscribers(self):
        """ Tests deleting of messages from a topic without subscribers.
        """
        ps = RedisPubSub(self.kvdb, self.key_prefix)

        msg_value = '"msg_value"'

        topic = Topic('/test/delete')
        ps.add_topic(topic)

        producer = Client('Producer', 'producer')
        ps.add_producer(producer, topic)

        pub_ctx = PubCtx()
        pub_ctx.client_id = producer.id
        pub_ctx.topic = topic.name
        pub_ctx.msg = Message(msg_value)

        msg_id = ps.publish(pub_ctx).msg.msg_id

        # We've got a msg_id now and we know this is a topic with no subsribers. In that case, deleting a messages should
        # also delete all the metadata related to it along with its payload.

        # First, let's confirm the messages is actually there

        result = self.kvdb.zrange(ps.MSG_IDS_PREFIX.format(topic.name), 0, -1)
        eq_(result, [msg_id])

        result = self.kvdb.hgetall(ps.MSG_VALUES_KEY)
        eq_(len(result), 1)
        self.assertTrue(msg_id in result)

        result = self.kvdb.hgetall(ps.MSG_METADATA_KEY)
        eq_(len(result), 1)
        self.assertTrue(msg_id in result)

        result = self.kvdb.hgetall(ps.MSG_EXPIRE_AT_KEY)
        eq_(len(result), 1)
        self.assertTrue(msg_id in result)

        # Ok, now delete the message and confirm it's not in Redis anymore

        ps.delete_from_topic(topic.name, msg_id)

        result = self.kvdb.zrange(ps.MSG_IDS_PREFIX.format(topic.name), 0, -1)
        eq_(result, [])

        result = self.kvdb.hgetall(ps.MSG_VALUES_KEY)
        eq_({}, result)

        result = self.kvdb.hgetall(ps.MSG_METADATA_KEY)
        eq_({}, result)

        result = self.kvdb.hgetall(ps.MSG_EXPIRE_AT_KEY)
        eq_({}, result)
Esempio n. 5
0
 def setUp(self):
     super(RedisPubSubTestCase, self).setUp()
     self.api = PubSubAPI(RedisPubSub(self.kvdb, self.key_prefix))
    def test_get_callback_consumers(self):
        ps = RedisPubSub(self.kvdb, self.key_prefix)

        topic = Topic('/test/delete')
        ps.add_topic(topic)

        producer = Client('Producer', 'producer')
        ps.add_producer(producer, topic)

        id1 = 'Consumer CB1'
        name1 = 'consumer-cb1'
        sub_key1 = new_cid()
        callback_id1 = rand_int()

        id2 = 'Consumer CB2'
        name2 = 'consumer-cb2'
        sub_key2 = new_cid()
        callback_id2 = rand_int()

        consumer_cb1 = Consumer(id1, name1, sub_key=sub_key1, delivery_mode=PUB_SUB.DELIVERY_MODE.CALLBACK_URL.id,
            callback_id=callback_id1)

        consumer_cb2 = Consumer(id2, name2, sub_key=sub_key2, delivery_mode=PUB_SUB.DELIVERY_MODE.CALLBACK_URL.id,
            callback_id=callback_id2)

        consumer_pull = Consumer('Consumer pull', 'consumer-pull', sub_key=new_cid(), delivery_mode=PUB_SUB.DELIVERY_MODE.PULL.id)
        consumer_inactive = Consumer(
            'Consumer pull', 'consumer-pull', is_active=False, sub_key=new_cid(), delivery_mode=PUB_SUB.DELIVERY_MODE.PULL.id)

        ps.add_consumer(consumer_cb1, topic)
        ps.add_consumer(consumer_cb2, topic)
        ps.add_consumer(consumer_pull, topic)     # This one should not be returned because it's a pull one
        ps.add_consumer(consumer_inactive, topic) # This one should not be returned because it's inactive

        consumers = list(ps.get_callback_consumers())

        # Only 2 are returned, the rest won't make it
        eq_(len(consumers), 2)

        # Sort by each consumer's ID, i.e. in lexicographical order
        consumers.sort(key=attrgetter('id'))

        consumer1 = consumers[0]
        eq_(consumer1.id, id1)
        eq_(consumer1.name, name1)
        eq_(consumer1.is_active, True)
        eq_(consumer1.sub_key, sub_key1)
        eq_(consumer1.callback_id, callback_id1)

        consumer2 = consumers[1]
        eq_(consumer2.id, id2)
        eq_(consumer2.name, name2)
        eq_(consumer2.is_active, True)
        eq_(consumer2.sub_key, sub_key2)
        eq_(consumer2.callback_id, callback_id2)
    def test_ack_delete_from(self):
        """ Tests ack and delete on a consumer's queue.
        """
        # Both ack and delete use the same implementation and it's only the user-exposed API that differs.

        ps = RedisPubSub(self.kvdb, self.key_prefix)

        msg_value = '"msg_value"'

        topic = Topic('/test/delete')
        ps.add_topic(topic)

        producer = Client('Producer', 'producer')
        ps.add_producer(producer, topic)

        consumer = Consumer('Consumer', 'consumer', sub_key=new_cid())
        ps.add_consumer(consumer, topic)

        pub_ctx = PubCtx()
        pub_ctx.client_id = producer.id
        pub_ctx.topic = topic.name
        pub_ctx.msg = Message(msg_value)

        msg_id = ps.publish(pub_ctx).msg.msg_id
        ps.move_to_target_queues()

        # A message has been published and move to a consumer's queue so let's confirm the fact first.
        result = self.kvdb.lrange(ps.CONSUMER_MSG_IDS_PREFIX.format(consumer.sub_key), 0, -1)
        eq_(result, [msg_id])

        # Ok, now delete the message and confirm it's not in the consumer's queue anymore.

        ps.delete_from_consumer_queue(consumer.sub_key, [msg_id])

        result = self.kvdb.lrange(ps.CONSUMER_MSG_IDS_PREFIX.format(consumer.sub_key), 0, -1)
        eq_(result, [])
    def test_delete_from_topic_has_subscribers(self):
        """ Tests deleting of messages from a topic which has subscribers.
        """
        ps = RedisPubSub(self.kvdb, self.key_prefix)

        msg_value = '"msg_value"'

        topic = Topic('/test/delete')
        ps.add_topic(topic)

        producer = Client('Producer', 'producer')
        ps.add_producer(producer, topic)

        consumer = Consumer('Consumer', 'consumer', new_cid())
        ps.add_consumer(consumer, topic)

        pub_ctx = PubCtx()
        pub_ctx.client_id = producer.id
        pub_ctx.topic = topic.name
        pub_ctx.msg = Message(msg_value)

        msg_id = ps.publish(pub_ctx).msg.msg_id

        # We've got a msg_id now and we know this is a topic which has a subscriber so deleting a message from the topic
        # should not delete it from any other place because the consumer may still hold onto it.

        # First, let's confirm the messages is actually there

        result = self.kvdb.zrange(ps.MSG_IDS_PREFIX.format(topic.name), 0, -1)
        eq_(result, [msg_id])

        result = self.kvdb.hgetall(ps.MSG_VALUES_KEY)
        eq_(len(result), 1)
        self.assertTrue(msg_id in result)

        result = self.kvdb.hgetall(ps.MSG_METADATA_KEY)
        eq_(len(result), 1)
        self.assertTrue(msg_id in result)

        result = self.kvdb.hgetall(ps.MSG_EXPIRE_AT_KEY)
        eq_(len(result), 1)
        self.assertTrue(msg_id in result)

        # Ok, now delete the message and confirm it's not in the topic anymore however it's still kept elsewhere.

        ps.delete_from_topic(topic.name, msg_id)

        result = self.kvdb.zrange(ps.MSG_IDS_PREFIX.format(topic.name), 0, -1)
        eq_(result, [])

        result = self.kvdb.hgetall(ps.MSG_VALUES_KEY)
        eq_(len(result), 1)
        self.assertTrue(msg_id in result)

        result = self.kvdb.hgetall(ps.MSG_METADATA_KEY)
        eq_(len(result), 1)
        self.assertTrue(msg_id in result)

        result = self.kvdb.hgetall(ps.MSG_EXPIRE_AT_KEY)
        eq_(len(result), 1)
        self.assertTrue(msg_id in result)
    def test_full_path(self):
        """ Tests full sub/pub/ack/reject path with 4 topics and 3 clients. Doesn't test background tasks.
        """
        # Give up early if there is no connection to Redis at all
        if not self.has_redis:
            return

        """ What is tested.

        - 3 clients connect to pub/sub: CRM, Billing and ERP.
        - 4 topics are created: /cust/new, /cust/update, /adsl/new and /adsl/update

        - Subscriptions are:

          - CRM subs to /adsl/new
          - CRM subs to /adsl/update

          - Billing subs to /cust/new
          - Billing subs to /cust/update

          - ERP subs to /adsl/new
          - ERP subs to /adsl/update

        - Publications are:

          - CRM publishes Msg-CRM1 to /cust/new -------------- TTL of 0.1s
          - CRM publishes Msg-CRM2 to /cust/update ----------- TTL of 0.2s

          - Billing publishes Msg-Billing1 to /adsl/new ------ TTL of 0.3s
          - Billing publishes Msg-Billing2 to /adsl/update --- TTL of 3600s

          - (ERP doesn't publish anything)

        - Expected deliveries are:

          - Msg-CRM1 goes to Billing
          - Msg-CRM2 goes to Billing

          - Msg-Billing1 goes to CRM
          - Msg-Billing2 goes to CRM

          - Msg-Billing1 goes to ERP
          - Msg-Billing2 goes to ERP

        - Confirmations are:

          - CRM acks Msg-Billing1
          - CRM rejects Msg-Billing2

          - Billing acks Msg-CRM1
          - Billing acks Msg-CRM2

          - ERP rejects Msg-Billing1
          - ERP acks Msg-Billing2

        - Clean up tasks are

          - Msg-CRM1 is deleted because it's confirmed by its only recipient of Billing
          - Msg-CRM2 is deleted because it's confirmed by its only recipient of Billing

          - Msg-Billing1 is deleted because:

            - CRM confirms it
            - ERP rejects it but the message's TTL is 0.3s so it times out
              (In real world ERP would have possibly acknowledged it in say, 2s, but it would've been too late)

          - Msg-Billing2 is not deleted because:

            - ERP confirms it
            - CRM rejects it and the message's TTL is 3600s so it's still around when a clean up task runs

        """

        ps = RedisPubSub(self.kvdb, self.key_prefix)

        msg_billing1_value = '"msg_billing1"'
        msg_billing2_value = '"msg_billing2"'

        msg_crm1_value = '"msg_crm1"'
        msg_crm2_value = '"msg_crm2"'

        # Check all the Lua programs are loaded

        eq_(len(ps.lua_programs), 9)

        for attr in dir(ps):
            if attr.startswith('LUA'):
                value = getattr(ps, attr)
                self.assertTrue(value in ps.lua_programs, value)

        topic_cust_new = Topic('/cust/new')
        topic_cust_update = Topic('/cust/update')

        topic_adsl_new = Topic('/adsl/new')
        topic_adsl_update = Topic('/adsl/update')

        ps.add_topic(topic_cust_new)
        ps.add_topic(topic_cust_update)

        ps.add_topic(topic_adsl_new)
        ps.add_topic(topic_adsl_update)

        # Check all the topics are cached locally

        eq_(len(ps.topics), 4)

        for topic in(topic_cust_new, topic_cust_update, topic_adsl_new, topic_adsl_update):
            eq_(ps.topics[topic.name], topic)

        client_id_crm = Consumer('CRM', 'CRM', sub_key='sub_key_crm')
        client_id_billing = Consumer('Billing', 'Billing', sub_key='sub_key_billing')
        client_id_erp = Consumer('ERP', 'ERP', sub_key='sub_key_crm')

        ps.add_producer(client_id_crm, topic_cust_new)
        ps.add_producer(client_id_crm, topic_cust_update)

        ps.add_producer(client_id_billing, topic_adsl_new)
        ps.add_producer(client_id_billing, topic_adsl_update)

        ps.add_consumer(client_id_crm, topic_adsl_new)
        ps.add_consumer(client_id_crm, topic_adsl_update)

        ps.add_consumer(client_id_billing, topic_cust_new)
        ps.add_consumer(client_id_billing, topic_cust_update)

        ps.add_consumer(client_id_erp, topic_adsl_new)
        ps.add_consumer(client_id_erp, topic_adsl_update)

        # Check producers have been registered for topics

        eq_(len(ps.prod_to_topic), 2)

        self.assertTrue(client_id_crm.id in ps.prod_to_topic)
        self.assertTrue(client_id_billing.id in ps.prod_to_topic)
        self.assertTrue(client_id_erp.id not in ps.prod_to_topic)

        self.assertTrue(isinstance(ps.prod_to_topic[client_id_crm.id], set))
        eq_(sorted(ps.prod_to_topic[client_id_crm.id]), ['/cust/new', '/cust/update'])

        self.assertTrue(isinstance(ps.prod_to_topic[client_id_billing.id], set))
        eq_(sorted(ps.prod_to_topic[client_id_billing.id]), ['/adsl/new', '/adsl/update'])

        # Subscribe all the systems

        sub_ctx_crm = SubCtx()
        sub_ctx_crm.client_id = client_id_crm.id
        sub_ctx_crm.topics = [topic_adsl_new.name, topic_adsl_update.name]

        sub_ctx_billing = SubCtx()
        sub_ctx_billing.client_id = client_id_billing.id
        sub_ctx_billing.topics = [topic_cust_new.name, topic_cust_update.name]

        sub_ctx_erp = SubCtx()
        sub_ctx_erp.client_id = client_id_erp.id
        sub_ctx_erp.topics = [topic_adsl_new.name, topic_adsl_update.name]

        sub_key_crm = 'sub_key_crm'
        sub_key_billing = 'sub_key_billing'
        sub_key_erp = 'sub_key_erp'

        received_sub_key_crm = ps.subscribe(sub_ctx_crm, sub_key_crm)
        received_sub_key_billing = ps.subscribe(sub_ctx_billing, sub_key_billing)
        received_sub_key_erp = ps.subscribe(sub_ctx_erp, sub_key_erp)

        eq_(sub_key_crm, received_sub_key_crm)
        eq_(sub_key_billing, received_sub_key_billing)
        eq_(sub_key_erp, received_sub_key_erp)

        eq_(sorted(ps.sub_to_cons.items()), [('sub_key_billing', 'Billing'), ('sub_key_crm', 'CRM'), ('sub_key_erp', 'ERP')])
        eq_(sorted(ps.cons_to_sub.items()), [('Billing', 'sub_key_billing'), ('CRM', 'sub_key_crm'), ('ERP', 'sub_key_erp')])

        # CRM publishes Msg-CRM1 to /cust/new
        pub_ctx_msg_crm1 = PubCtx()
        pub_ctx_msg_crm1.client_id = client_id_crm.id
        pub_ctx_msg_crm1.topic = topic_cust_new.name
        pub_ctx_msg_crm1.msg = Message(msg_crm1_value, mime_type='text/xml', priority=1, expiration=0.1)

        msg_crm1_id = ps.publish(pub_ctx_msg_crm1).msg.msg_id

        # CRM publishes Msg-CRM2 to /cust/new
        pub_ctx_msg_crm2 = PubCtx()
        pub_ctx_msg_crm2.client_id = client_id_crm.id
        pub_ctx_msg_crm2.topic = topic_cust_update.name
        pub_ctx_msg_crm2.msg = Message(msg_crm2_value, mime_type='application/json', priority=2, expiration=0.2)

        msg_crm2_id = ps.publish(pub_ctx_msg_crm2).msg.msg_id

        # Billing publishes Msg-Billing1 to /adsl/new
        pub_ctx_msg_billing1 = PubCtx()
        pub_ctx_msg_billing1.client_id = client_id_billing.id
        pub_ctx_msg_billing1.topic = topic_adsl_new.name
        pub_ctx_msg_billing1.msg = Message(msg_billing1_value, mime_type='application/soap+xml', priority=3, expiration=0.3)

        msg_billing1_id = ps.publish(pub_ctx_msg_billing1).msg.msg_id

        # Billing publishes Msg-Billing2 to /adsl/update
        pub_ctx_msg_billing2 = PubCtx()
        pub_ctx_msg_billing2.client_id = client_id_billing.id
        pub_ctx_msg_billing2.topic = topic_adsl_update.name

        # Nothing except payload and expiration set, defaults should be used
        msg_billing2 = Message(msg_billing2_value, expiration=3600)
        pub_ctx_msg_billing2.msg = msg_billing2

        msg_billing2_id = ps.publish(pub_ctx_msg_billing2).msg.msg_id

        keys = self.kvdb.keys('{}*'.format(self.key_prefix))
        eq_(len(keys), 9)

        expected_keys = [ps.MSG_VALUES_KEY, ps.MSG_EXPIRE_AT_KEY, ps.LAST_PUB_TIME_KEY]
        for topic in topic_cust_new, topic_cust_update, topic_adsl_new, topic_adsl_update:
            expected_keys.append(ps.MSG_IDS_PREFIX.format(topic.name))

        for key in expected_keys:
            self.assertIn(key, keys)

        # Check values of messages published
        self._check_msg_values_metadata(ps, msg_crm1_id, msg_crm2_id, msg_billing1_id, msg_billing2_id, True)

        # Now move the messages just published to each of the subscriber's queue.
        # In a real environment this is done by a background job.
        ps.move_to_target_queues()

        # Now all the messages have been moved we can check if everything is in place
        # ready for subscribers to get their messages.

        keys = self.kvdb.keys('{}*'.format(self.key_prefix))
        eq_(len(keys), 9)

        self.assertIn(ps.UNACK_COUNTER_KEY, keys)
        self.assertIn(ps.MSG_VALUES_KEY, keys)

        for sub_key in(sub_key_crm, sub_key_billing, sub_key_erp):
            key = ps.CONSUMER_MSG_IDS_PREFIX.format(sub_key)
            self.assertIn(key, keys)

        self._check_unack_counter(ps, msg_crm1_id, msg_crm2_id, msg_billing1_id, msg_billing2_id, 1, 1, 2, 2)

        # Check values of messages published are still there
        self._check_msg_values_metadata(ps, msg_crm1_id, msg_crm2_id, msg_billing1_id, msg_billing2_id, True)

        # Check that each recipient has expected message IDs in its respective message queue
        keys = self.kvdb.keys(ps.CONSUMER_MSG_IDS_PREFIX.format('*'))
        eq_(len(keys), 3)

        msg_ids_crm = self.kvdb.lrange(ps.CONSUMER_MSG_IDS_PREFIX.format(sub_key_crm), 0, 100)
        msg_ids_billing = self.kvdb.lrange(ps.CONSUMER_MSG_IDS_PREFIX.format(sub_key_billing), 0, 100)
        msg_ids_erp = self.kvdb.lrange(ps.CONSUMER_MSG_IDS_PREFIX.format(sub_key_erp), 0, 100)

        eq_(len(msg_ids_crm), 2)
        eq_(len(msg_ids_billing), 2)
        eq_(len(msg_ids_erp), 2)

        self.assertIn(msg_billing1_id, msg_ids_crm)
        self.assertIn(msg_billing2_id, msg_ids_crm)

        self.assertIn(msg_billing1_id, msg_ids_erp)
        self.assertIn(msg_billing2_id, msg_ids_erp)

        self.assertIn(msg_crm1_id, msg_ids_billing)
        self.assertIn(msg_crm1_id, msg_ids_billing)

        # Now that the messages are in queues, let's fetch them

        get_ctx_crm = GetCtx()
        get_ctx_crm.sub_key = sub_key_crm

        get_ctx_billing = GetCtx()
        get_ctx_billing.sub_key = sub_key_billing

        get_ctx_erp = GetCtx()
        get_ctx_erp.sub_key = sub_key_erp

        msgs_crm = sorted(list(ps.get(get_ctx_crm)), key=attrgetter('payload'))
        msgs_billing = sorted(list(ps.get(get_ctx_billing)), key=attrgetter('payload'))
        msgs_erp = sorted(list(ps.get(get_ctx_erp)), key=attrgetter('payload'))

        eq_(len(msgs_crm), 2)
        eq_(len(msgs_billing), 2)
        eq_(len(msgs_erp), 2)

        self._assert_has_msg_metadata(msgs_crm, msg_billing1_id, 'application/soap+xml', 3, 0.3)
        self._assert_has_msg_metadata(msgs_crm, msg_billing2_id, 'text/plain', 5, 3600)

        self._assert_has_msg_metadata(msgs_billing, msg_crm1_id, 'text/xml', 1, 0.1)
        self._assert_has_msg_metadata(msgs_billing, msg_crm2_id, 'application/json', 2, 0.2)

        self._assert_has_msg_metadata(msgs_erp, msg_billing1_id, 'application/soap+xml', 3, 0.3)
        self._assert_has_msg_metadata(msgs_erp, msg_billing2_id, 'text/plain', 5, 3600)

        # Check in-flight status for each message got
        keys = self.kvdb.keys(ps.CONSUMER_IN_FLIGHT_DATA_PREFIX.format('*'))
        eq_(len(keys), 3)

        now = arrow.utcnow()

        self._check_in_flight(ps, now, sub_key_crm, sub_key_billing, sub_key_erp, msg_crm1_id, msg_crm2_id,
                msg_billing1_id, msg_billing2_id, True, True, True, True, True, True)

        # Messages should still be undelivered hence their unack counters are not touched at this point
        self._check_unack_counter(ps, msg_crm1_id, msg_crm2_id, msg_billing1_id, msg_billing2_id, 1, 1, 2, 2)

# ######################################################################################################################

        # Messages are fetched, they can be confirmed or rejected now

        # CRM
        ack_ctx_crm = AckCtx()
        ack_ctx_crm.sub_key = sub_key_crm
        ack_ctx_crm.append(msg_billing1_id)

        reject_ctx_crm = RejectCtx()
        reject_ctx_crm.sub_key = sub_key_crm
        reject_ctx_crm.append(msg_billing2_id)

        ps.acknowledge_delete(ack_ctx_crm)
        ps.reject(reject_ctx_crm)

        # One in-flight less
        self._check_in_flight(ps, now, sub_key_crm, sub_key_billing, sub_key_erp, msg_crm1_id, msg_crm2_id,
                msg_billing1_id, msg_billing2_id, False, False, True, True, True, True)

        # Rejections, as with reject_ctx_crm, don't change unack count
        self._check_unack_counter(ps, msg_crm1_id, msg_crm2_id, msg_billing1_id, msg_billing2_id, 1, 1, 1, 2)

        # Billing
        ack_ctx_billing = AckCtx()
        ack_ctx_billing.sub_key = sub_key_billing
        ack_ctx_billing.append(msg_crm1_id)
        ack_ctx_billing.append(msg_crm2_id)

        ps.acknowledge_delete(ack_ctx_billing)

        # Two in-flight less
        self._check_in_flight(ps, now, sub_key_crm, sub_key_billing, sub_key_erp, msg_crm1_id, msg_crm2_id,
                msg_billing1_id, msg_billing2_id, False, False, False, False, True, True)

        # Again, rejections, as with reject_ctx_crm, don't change unack count
        self._check_unack_counter(ps, None, None, msg_billing1_id, msg_billing2_id, 1, 1, 1, 2)

        # ERP
        reject_ctx_erp = RejectCtx()
        reject_ctx_erp.sub_key = sub_key_erp
        reject_ctx_erp.append(msg_billing1_id)

        ack_ctx_erp = AckCtx()
        ack_ctx_erp.sub_key = sub_key_erp
        ack_ctx_erp.append(msg_billing2_id)

        ps.reject(reject_ctx_erp)
        ps.acknowledge_delete(ack_ctx_erp)

        # Another in-flight less
        self._check_in_flight(ps, now, sub_key_crm, sub_key_billing, sub_key_erp, msg_crm1_id, msg_crm2_id,
                msg_billing1_id, msg_billing2_id, False, False, False, False, False, False)

        # And again, rejections, as with reject_ctx_crm, don't change unack count
        self._check_unack_counter(ps, None, None, msg_billing1_id, msg_billing2_id, 1, 1, 1, 1)

        # Sleep for a moment to make sure enough time passes for messages to expire
        sleep(0.4)

        # Deletes everything except for Msg-Billing2 which has a TTL of 3600
        ps.delete_expired()

        keys = self.kvdb.keys('{}*'.format(self.key_prefix))
        eq_(len(keys), 8)

        expected_keys = [ps.MSG_VALUES_KEY, ps.MSG_EXPIRE_AT_KEY, ps.UNACK_COUNTER_KEY,
                         ps.CONSUMER_MSG_IDS_PREFIX.format(sub_key_crm)]
        for key in expected_keys:
            self.assertTrue(key in keys, 'Key not found `{}` in `{}`'.format(key, keys))

        # Check all the remaining keys - still concerning Msg-Billing2 only
        # because this is the only message that wasn't confirmed nor expired.

        expire_at = self.kvdb.hgetall(ps.MSG_EXPIRE_AT_KEY)
        eq_(len(expire_at), 1)
        eq_(expire_at[msg_billing2_id], msg_billing2.expire_at_utc.isoformat())

        unack_counter = self.kvdb.hgetall(ps.UNACK_COUNTER_KEY)
        eq_(len(unack_counter), 1)
        eq_(unack_counter[msg_billing2_id], '1')

        crm_messages = self.kvdb.lrange(ps.CONSUMER_MSG_IDS_PREFIX.format(sub_key_crm), 0, 100)
        eq_(len(crm_messages), 1)
        eq_(crm_messages[0],msg_billing2_id)

        msg_values = self.kvdb.hgetall(ps.MSG_METADATA_KEY)
        eq_(len(msg_values), 2)
        eq_(sorted(loads(msg_values[msg_billing2_id]).items()), sorted(msg_billing2.to_dict().items()))
Esempio n. 10
0
    def set_up_config(self, server):

        # Which components are enabled
        self.component_enabled.stats = asbool(self.fs_server_config.component_enabled.stats)
        self.component_enabled.slow_response = asbool(self.fs_server_config.component_enabled.slow_response)
        self.component_enabled.live_msg_browser = asbool(self.fs_server_config.component_enabled.live_msg_browser)

        # Details of what is enabled in live message browser
        self.live_msg_browser = self.fs_server_config.live_msg_browser
        self.live_msg_browser.include_internal = asbool(self.live_msg_browser.include_internal)

        # Pub/sub
        self.pubsub = PubSubAPI(RedisPubSub(self.kvdb.conn))

        #
        # Cassandra - start
        #

        query = self.odb.get_cassandra_conn_list(server.cluster.id, True)
        self.config.cassandra_conn = ConfigDict.from_query('cassandra_conn', query)

        query = self.odb.get_cassandra_query_list(server.cluster.id, True)
        self.config.cassandra_query = ConfigDict.from_query('cassandra_query', query)

        #
        # Cassandra - end
        #

        #
        # Search - start
        #

        query = self.odb.get_search_es_list(server.cluster.id, True)
        self.config.search_es = ConfigDict.from_query('search_es', query)

        query = self.odb.get_search_solr_list(server.cluster.id, True)
        self.config.search_solr = ConfigDict.from_query('search_solr', query)

        #
        # Search - end
        #

        #
        # Cloud - start
        #

        # OpenStack - Swift

        query = self.odb.get_cloud_openstack_swift_list(server.cluster.id, True)
        self.config.cloud_openstack_swift = ConfigDict.from_query('cloud_openstack_swift', query)

        query = self.odb.get_cloud_aws_s3_list(server.cluster.id, True)
        self.config.cloud_aws_s3 = ConfigDict.from_query('cloud_aws_s3', query)

        #
        # Cloud - end
        #

        # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

        # Services
        query = self.odb.get_service_list(server.cluster.id, True)
        self.config.service = ConfigDict.from_query('service_list', query)

        #
        # Definitions - start
        #

        # AMQP
        query = self.odb.get_definition_amqp_list(server.cluster.id, True)
        self.config.definition_amqp = ConfigDict.from_query('definition_amqp', query)

        #
        # Definitions - end
        #

        #
        # Channels - start
        #

        # AMQP
        query = self.odb.get_channel_amqp_list(server.cluster.id, True)
        self.config.channel_amqp = ConfigDict.from_query('channel_amqp', query)

        # STOMP
        query = self.odb.get_channel_stomp_list(server.cluster.id, True)
        self.config.channel_stomp = ConfigDict.from_query('channel_stomp', query)

        #
        # Channels - end
        #

        #
        # Outgoing connections - start
        #

        # AMQP
        query = self.odb.get_out_amqp_list(server.cluster.id, True)
        self.config.out_amqp = ConfigDict.from_query('out_amqp', query)

        # FTP
        query = self.odb.get_out_ftp_list(server.cluster.id, True)
        self.config.out_ftp = ConfigDict.from_query('out_ftp', query)

        # JMS WMQ
        query = self.odb.get_out_jms_wmq_list(server.cluster.id, True)
        self.config.out_jms_wmq = ConfigDict.from_query('out_jms_wmq', query)

        # Odoo
        query = self.odb.get_out_odoo_list(server.cluster.id, True)
        self.config.out_odoo = ConfigDict.from_query('out_odoo', query)

        # Plain HTTP
        query = self.odb.get_http_soap_list(server.cluster.id, 'outgoing', 'plain_http', True)
        self.config.out_plain_http = ConfigDict.from_query('out_plain_http', query)

        # SOAP
        query = self.odb.get_http_soap_list(server.cluster.id, 'outgoing', 'soap', True)
        self.config.out_soap = ConfigDict.from_query('out_soap', query)

        # SQL
        query = self.odb.get_out_sql_list(server.cluster.id, True)
        self.config.out_sql = ConfigDict.from_query('out_sql', query)

        # STOMP
        query = self.odb.get_out_stomp_list(server.cluster.id, True)
        self.config.out_stomp = ConfigDict.from_query('out_stomp', query)

        # ZMQ channels
        query = self.odb.get_channel_zmq_list(server.cluster.id, True)
        self.config.channel_zmq = ConfigDict.from_query('channel_zmq', query)

        # ZMQ outgoing
        query = self.odb.get_out_zmq_list(server.cluster.id, True)
        self.config.out_zmq = ConfigDict.from_query('out_zmq', query)

        # WebSocket channels
        query = self.odb.get_channel_web_socket_list(server.cluster.id, True)
        self.config.channel_web_socket = ConfigDict.from_query('channel_web_socket', query)

        #
        # Outgoing connections - end
        #

        # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

        #
        # Notifications - start
        #

        # OpenStack Swift
        query = self.odb.get_notif_cloud_openstack_swift_list(server.cluster.id, True)
        self.config.notif_cloud_openstack_swift = ConfigDict.from_query('notif_cloud_openstack_swift', query)

        # SQL
        query = self.odb.get_notif_sql_list(server.cluster.id, True)
        self.config.notif_sql = ConfigDict.from_query('notif_sql', query)

        #
        # Notifications - end
        #

        # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

        #
        # Security - start
        #

        # API keys
        query = self.odb.get_apikey_security_list(server.cluster.id, True)
        self.config.apikey = ConfigDict.from_query('apikey', query)

        # AWS
        query = self.odb.get_aws_security_list(server.cluster.id, True)
        self.config.aws = ConfigDict.from_query('aws', query)

        # HTTP Basic Auth
        query = self.odb.get_basic_auth_list(server.cluster.id, None, True)
        self.config.basic_auth = ConfigDict.from_query('basic_auth', query)

        # HTTP Basic Auth
        query = self.odb.get_jwt_list(server.cluster.id, None, True)
        self.config.jwt = ConfigDict.from_query('jwt', query)

        # NTLM
        query = self.odb.get_ntlm_list(server.cluster.id, True)
        self.config.ntlm = ConfigDict.from_query('ntlm', query)

        # OAuth
        query = self.odb.get_oauth_list(server.cluster.id, True)
        self.config.oauth = ConfigDict.from_query('oauth', query)

        # OpenStack
        query = self.odb.get_openstack_security_list(server.cluster.id, True)
        self.config.openstack_security = ConfigDict.from_query('openstack_security', query)

        # RBAC - permissions
        query = self.odb.get_rbac_permission_list(server.cluster.id, True)
        self.config.rbac_permission = ConfigDict.from_query('rbac_permission', query)

        # RBAC - roles
        query = self.odb.get_rbac_role_list(server.cluster.id, True)
        self.config.rbac_role = ConfigDict.from_query('rbac_role', query)

        # RBAC - client roles
        query = self.odb.get_rbac_client_role_list(server.cluster.id, True)
        self.config.rbac_client_role = ConfigDict.from_query('rbac_client_role', query)

        # RBAC - role permission
        query = self.odb.get_rbac_role_permission_list(server.cluster.id, True)
        self.config.rbac_role_permission = ConfigDict.from_query('rbac_role_permission', query)

        # Technical accounts
        query = self.odb.get_tech_acc_list(server.cluster.id, True)
        self.config.tech_acc = ConfigDict.from_query('tech_acc', query)

        # TLS CA certs
        query = self.odb.get_tls_ca_cert_list(server.cluster.id, True)
        self.config.tls_ca_cert = ConfigDict.from_query('tls_ca_cert', query)

        # TLS channel security
        query = self.odb.get_tls_channel_sec_list(server.cluster.id, True)
        self.config.tls_channel_sec = ConfigDict.from_query('tls_channel_sec', query)

        # TLS key/cert pairs
        query = self.odb.get_tls_key_cert_list(server.cluster.id, True)
        self.config.tls_key_cert = ConfigDict.from_query('tls_key_cert', query)

        # WS-Security
        query = self.odb.get_wss_list(server.cluster.id, True)
        self.config.wss = ConfigDict.from_query('wss', query)

        # Vault connections
        query = self.odb.get_vault_connection_list(server.cluster.id, True)
        self.config.vault_conn_sec = ConfigDict.from_query('vault_conn_sec', query)

        # XPath
        query = self.odb.get_xpath_sec_list(server.cluster.id, True)
        self.config.xpath_sec = ConfigDict.from_query('xpath_sec', query)

        #
        # Security - end
        #

        # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

        # All the HTTP/SOAP channels.
        http_soap = []
        for item in self.odb.get_http_soap_list(server.cluster.id, 'channel'):

            hs_item = {}
            for key in item.keys():
                hs_item[key] = getattr(item, key)

            hs_item['replace_patterns_json_pointer'] = item.replace_patterns_json_pointer
            hs_item['replace_patterns_xpath'] = item.replace_patterns_xpath

            hs_item['match_target'] = '{}{}{}'.format(hs_item['soap_action'], MISC.SEPARATOR, hs_item['url_path'])
            hs_item['match_target_compiled'] = Matcher(hs_item['match_target'])

            http_soap.append(hs_item)

        self.config.http_soap = http_soap

        # Namespaces
        query = self.odb.get_namespace_list(server.cluster.id, True)
        self.config.msg_ns = ConfigDict.from_query('msg_ns', query)

        # XPath
        query = self.odb.get_xpath_list(server.cluster.id, True)
        self.config.xpath = ConfigDict.from_query('msg_xpath', query)

        # JSON Pointer
        query = self.odb.get_json_pointer_list(server.cluster.id, True)
        self.config.json_pointer = ConfigDict.from_query('json_pointer', query)

        # SimpleIO
        self.config.simple_io = ConfigDict('simple_io', Bunch())
        self.config.simple_io['int_parameters'] = self.int_parameters
        self.config.simple_io['int_parameter_suffixes'] = self.int_parameter_suffixes
        self.config.simple_io['bool_parameter_prefixes'] = self.bool_parameter_prefixes

        # Pub/sub config
        self.config.pubsub = Bunch()
        self.config.pubsub.default_consumer = Bunch()
        self.config.pubsub.default_producer = Bunch()

        query = self.odb.get_pubsub_topic_list(server.cluster.id, True)
        self.config.pubsub.topics = ConfigDict.from_query('pubsub_topics', query)

        id, name = self.odb.get_pubsub_default_client(server.cluster.id, 'zato.pubsub.default-consumer')
        self.config.pubsub.default_consumer.id, self.config.pubsub.default_consumer.name = id, name

        id, name = self.odb.get_pubsub_default_client(server.cluster.id, 'zato.pubsub.default-producer')
        self.config.pubsub.default_producer.id, self.config.pubsub.default_producer.name = id, name

        query = self.odb.get_pubsub_producer_list(server.cluster.id, True)
        self.config.pubsub.producers = ConfigDict.from_query('pubsub_producers', query, list_config=True)

        query = self.odb.get_pubsub_consumer_list(server.cluster.id, True)
        self.config.pubsub.consumers = ConfigDict.from_query('pubsub_consumers', query, list_config=True)

        # E-mail - SMTP
        query = self.odb.get_email_smtp_list(server.cluster.id, True)
        self.config.email_smtp = ConfigDict.from_query('email_smtp', query)

        # E-mail - IMAP
        query = self.odb.get_email_imap_list(server.cluster.id, True)
        self.config.email_imap = ConfigDict.from_query('email_imap', query)

        # Message paths
        self.config.msg_ns_store = NamespaceStore()
        self.config.json_pointer_store = JSONPointerStore()
        self.config.xpath_store = XPathStore()

        # Assign config to worker
        self.worker_store.worker_config = self.config
        self.worker_store.pubsub = self.pubsub
Esempio n. 11
0
    def test_get_callback_consumers(self):
        ps = RedisPubSub(self.kvdb, self.key_prefix)

        topic = Topic('/test/delete')
        ps.add_topic(topic)

        producer = Client('Producer', 'producer')
        ps.add_producer(producer, topic)

        id1 = 'Consumer CB1'
        name1 = 'consumer-cb1'
        sub_key1 = new_cid()
        callback_id1 = rand_int()

        id2 = 'Consumer CB2'
        name2 = 'consumer-cb2'
        sub_key2 = new_cid()
        callback_id2 = rand_int()

        consumer_cb1 = Consumer(
            id1,
            name1,
            sub_key=sub_key1,
            delivery_mode=PUB_SUB.DELIVERY_MODE.CALLBACK_URL.id,
            callback_id=callback_id1)

        consumer_cb2 = Consumer(
            id2,
            name2,
            sub_key=sub_key2,
            delivery_mode=PUB_SUB.DELIVERY_MODE.CALLBACK_URL.id,
            callback_id=callback_id2)

        consumer_pull = Consumer('Consumer pull',
                                 'consumer-pull',
                                 sub_key=new_cid(),
                                 delivery_mode=PUB_SUB.DELIVERY_MODE.PULL.id)
        consumer_inactive = Consumer(
            'Consumer pull',
            'consumer-pull',
            is_active=False,
            sub_key=new_cid(),
            delivery_mode=PUB_SUB.DELIVERY_MODE.PULL.id)

        ps.add_consumer(consumer_cb1, topic)
        ps.add_consumer(consumer_cb2, topic)
        ps.add_consumer(
            consumer_pull,
            topic)  # This one should not be returned because it's a pull one
        ps.add_consumer(
            consumer_inactive,
            topic)  # This one should not be returned because it's inactive

        consumers = list(ps.get_callback_consumers())

        # Only 2 are returned, the rest won't make it
        eq_(len(consumers), 2)

        # Sort by each consumer's ID, i.e. in lexicographical order
        consumers.sort(key=attrgetter('id'))

        consumer1 = consumers[0]
        eq_(consumer1.id, id1)
        eq_(consumer1.name, name1)
        eq_(consumer1.is_active, True)
        eq_(consumer1.sub_key, sub_key1)
        eq_(consumer1.callback_id, callback_id1)

        consumer2 = consumers[1]
        eq_(consumer2.id, id2)
        eq_(consumer2.name, name2)
        eq_(consumer2.is_active, True)
        eq_(consumer2.sub_key, sub_key2)
        eq_(consumer2.callback_id, callback_id2)
Esempio n. 12
0
    def test_ack_delete_from(self):
        """ Tests ack and delete on a consumer's queue.
        """
        # Both ack and delete use the same implementation and it's only the user-exposed API that differs.

        ps = RedisPubSub(self.kvdb, self.key_prefix)

        msg_value = '"msg_value"'

        topic = Topic('/test/delete')
        ps.add_topic(topic)

        producer = Client('Producer', 'producer')
        ps.add_producer(producer, topic)

        consumer = Consumer('Consumer', 'consumer', sub_key=new_cid())
        ps.add_consumer(consumer, topic)

        pub_ctx = PubCtx()
        pub_ctx.client_id = producer.id
        pub_ctx.topic = topic.name
        pub_ctx.msg = Message(msg_value)

        msg_id = ps.publish(pub_ctx).msg.msg_id
        ps.move_to_target_queues()

        # A message has been published and move to a consumer's queue so let's confirm the fact first.
        result = self.kvdb.lrange(
            ps.CONSUMER_MSG_IDS_PREFIX.format(consumer.sub_key), 0, -1)
        eq_(result, [msg_id])

        # Ok, now delete the message and confirm it's not in the consumer's queue anymore.

        ps.delete_from_consumer_queue(consumer.sub_key, [msg_id])

        result = self.kvdb.lrange(
            ps.CONSUMER_MSG_IDS_PREFIX.format(consumer.sub_key), 0, -1)
        eq_(result, [])
Esempio n. 13
0
    def test_delete_from_topic_has_subscribers(self):
        """ Tests deleting of messages from a topic which has subscribers.
        """
        ps = RedisPubSub(self.kvdb, self.key_prefix)

        msg_value = '"msg_value"'

        topic = Topic('/test/delete')
        ps.add_topic(topic)

        producer = Client('Producer', 'producer')
        ps.add_producer(producer, topic)

        consumer = Consumer('Consumer', 'consumer', new_cid())
        ps.add_consumer(consumer, topic)

        pub_ctx = PubCtx()
        pub_ctx.client_id = producer.id
        pub_ctx.topic = topic.name
        pub_ctx.msg = Message(msg_value)

        msg_id = ps.publish(pub_ctx).msg.msg_id

        # We've got a msg_id now and we know this is a topic which has a subscriber so deleting a message from the topic
        # should not delete it from any other place because the consumer may still hold onto it.

        # First, let's confirm the messages is actually there

        result = self.kvdb.zrange(ps.MSG_IDS_PREFIX.format(topic.name), 0, -1)
        eq_(result, [msg_id])

        result = self.kvdb.hgetall(ps.MSG_VALUES_KEY)
        eq_(len(result), 1)
        self.assertTrue(msg_id in result)

        result = self.kvdb.hgetall(ps.MSG_METADATA_KEY)
        eq_(len(result), 1)
        self.assertTrue(msg_id in result)

        result = self.kvdb.hgetall(ps.MSG_EXPIRE_AT_KEY)
        eq_(len(result), 1)
        self.assertTrue(msg_id in result)

        # Ok, now delete the message and confirm it's not in the topic anymore however it's still kept elsewhere.

        ps.delete_from_topic(topic.name, msg_id)

        result = self.kvdb.zrange(ps.MSG_IDS_PREFIX.format(topic.name), 0, -1)
        eq_(result, [])

        result = self.kvdb.hgetall(ps.MSG_VALUES_KEY)
        eq_(len(result), 1)
        self.assertTrue(msg_id in result)

        result = self.kvdb.hgetall(ps.MSG_METADATA_KEY)
        eq_(len(result), 1)
        self.assertTrue(msg_id in result)

        result = self.kvdb.hgetall(ps.MSG_EXPIRE_AT_KEY)
        eq_(len(result), 1)
        self.assertTrue(msg_id in result)
Esempio n. 14
0
    def test_full_path(self):
        """ Tests full sub/pub/ack/reject path with 4 topics and 3 clients. Doesn't test background tasks.
        """
        # Give up early if there is no connection to Redis at all
        if not self.has_redis:
            return
        """ What is tested.

        - 3 clients connect to pub/sub: CRM, Billing and ERP.
        - 4 topics are created: /cust/new, /cust/update, /adsl/new and /adsl/update

        - Subscriptions are:

          - CRM subs to /adsl/new
          - CRM subs to /adsl/update

          - Billing subs to /cust/new
          - Billing subs to /cust/update

          - ERP subs to /adsl/new
          - ERP subs to /adsl/update

        - Publications are:

          - CRM publishes Msg-CRM1 to /cust/new -------------- TTL of 0.1s
          - CRM publishes Msg-CRM2 to /cust/update ----------- TTL of 0.2s

          - Billing publishes Msg-Billing1 to /adsl/new ------ TTL of 0.3s
          - Billing publishes Msg-Billing2 to /adsl/update --- TTL of 3600s

          - (ERP doesn't publish anything)

        - Expected deliveries are:

          - Msg-CRM1 goes to Billing
          - Msg-CRM2 goes to Billing

          - Msg-Billing1 goes to CRM
          - Msg-Billing2 goes to CRM

          - Msg-Billing1 goes to ERP
          - Msg-Billing2 goes to ERP

        - Confirmations are:

          - CRM acks Msg-Billing1
          - CRM rejects Msg-Billing2

          - Billing acks Msg-CRM1
          - Billing acks Msg-CRM2

          - ERP rejects Msg-Billing1
          - ERP acks Msg-Billing2

        - Clean up tasks are

          - Msg-CRM1 is deleted because it's confirmed by its only recipient of Billing
          - Msg-CRM2 is deleted because it's confirmed by its only recipient of Billing

          - Msg-Billing1 is deleted because:

            - CRM confirms it
            - ERP rejects it but the message's TTL is 0.3s so it times out
              (In real world ERP would have possibly acknowledged it in say, 2s, but it would've been too late)

          - Msg-Billing2 is not deleted because:

            - ERP confirms it
            - CRM rejects it and the message's TTL is 3600s so it's still around when a clean up task runs

        """

        ps = RedisPubSub(self.kvdb, self.key_prefix)

        msg_billing1_value = '"msg_billing1"'
        msg_billing2_value = '"msg_billing2"'

        msg_crm1_value = '"msg_crm1"'
        msg_crm2_value = '"msg_crm2"'

        # Check all the Lua programs are loaded

        eq_(len(ps.lua_programs), 9)

        for attr in dir(ps):
            if attr.startswith('LUA'):
                value = getattr(ps, attr)
                self.assertTrue(value in ps.lua_programs, value)

        topic_cust_new = Topic('/cust/new')
        topic_cust_update = Topic('/cust/update')

        topic_adsl_new = Topic('/adsl/new')
        topic_adsl_update = Topic('/adsl/update')

        ps.add_topic(topic_cust_new)
        ps.add_topic(topic_cust_update)

        ps.add_topic(topic_adsl_new)
        ps.add_topic(topic_adsl_update)

        # Check all the topics are cached locally

        eq_(len(ps.topics), 4)

        for topic in (topic_cust_new, topic_cust_update, topic_adsl_new,
                      topic_adsl_update):
            eq_(ps.topics[topic.name], topic)

        client_id_crm = Consumer('CRM', 'CRM', sub_key='sub_key_crm')
        client_id_billing = Consumer('Billing',
                                     'Billing',
                                     sub_key='sub_key_billing')
        client_id_erp = Consumer('ERP', 'ERP', sub_key='sub_key_crm')

        ps.add_producer(client_id_crm, topic_cust_new)
        ps.add_producer(client_id_crm, topic_cust_update)

        ps.add_producer(client_id_billing, topic_adsl_new)
        ps.add_producer(client_id_billing, topic_adsl_update)

        ps.add_consumer(client_id_crm, topic_adsl_new)
        ps.add_consumer(client_id_crm, topic_adsl_update)

        ps.add_consumer(client_id_billing, topic_cust_new)
        ps.add_consumer(client_id_billing, topic_cust_update)

        ps.add_consumer(client_id_erp, topic_adsl_new)
        ps.add_consumer(client_id_erp, topic_adsl_update)

        # Check producers have been registered for topics

        eq_(len(ps.prod_to_topic), 2)

        self.assertTrue(client_id_crm.id in ps.prod_to_topic)
        self.assertTrue(client_id_billing.id in ps.prod_to_topic)
        self.assertTrue(client_id_erp.id not in ps.prod_to_topic)

        self.assertTrue(isinstance(ps.prod_to_topic[client_id_crm.id], set))
        eq_(sorted(ps.prod_to_topic[client_id_crm.id]),
            ['/cust/new', '/cust/update'])

        self.assertTrue(isinstance(ps.prod_to_topic[client_id_billing.id],
                                   set))
        eq_(sorted(ps.prod_to_topic[client_id_billing.id]),
            ['/adsl/new', '/adsl/update'])

        # Subscribe all the systems

        sub_ctx_crm = SubCtx()
        sub_ctx_crm.client_id = client_id_crm.id
        sub_ctx_crm.topics = [topic_adsl_new.name, topic_adsl_update.name]

        sub_ctx_billing = SubCtx()
        sub_ctx_billing.client_id = client_id_billing.id
        sub_ctx_billing.topics = [topic_cust_new.name, topic_cust_update.name]

        sub_ctx_erp = SubCtx()
        sub_ctx_erp.client_id = client_id_erp.id
        sub_ctx_erp.topics = [topic_adsl_new.name, topic_adsl_update.name]

        sub_key_crm = 'sub_key_crm'
        sub_key_billing = 'sub_key_billing'
        sub_key_erp = 'sub_key_erp'

        received_sub_key_crm = ps.subscribe(sub_ctx_crm, sub_key_crm)
        received_sub_key_billing = ps.subscribe(sub_ctx_billing,
                                                sub_key_billing)
        received_sub_key_erp = ps.subscribe(sub_ctx_erp, sub_key_erp)

        eq_(sub_key_crm, received_sub_key_crm)
        eq_(sub_key_billing, received_sub_key_billing)
        eq_(sub_key_erp, received_sub_key_erp)

        eq_(sorted(ps.sub_to_cons.items()), [('sub_key_billing', 'Billing'),
                                             ('sub_key_crm', 'CRM'),
                                             ('sub_key_erp', 'ERP')])
        eq_(sorted(ps.cons_to_sub.items()), [('Billing', 'sub_key_billing'),
                                             ('CRM', 'sub_key_crm'),
                                             ('ERP', 'sub_key_erp')])

        # CRM publishes Msg-CRM1 to /cust/new
        pub_ctx_msg_crm1 = PubCtx()
        pub_ctx_msg_crm1.client_id = client_id_crm.id
        pub_ctx_msg_crm1.topic = topic_cust_new.name
        pub_ctx_msg_crm1.msg = Message(msg_crm1_value,
                                       mime_type='text/xml',
                                       priority=1,
                                       expiration=0.1)

        msg_crm1_id = ps.publish(pub_ctx_msg_crm1).msg.msg_id

        # CRM publishes Msg-CRM2 to /cust/new
        pub_ctx_msg_crm2 = PubCtx()
        pub_ctx_msg_crm2.client_id = client_id_crm.id
        pub_ctx_msg_crm2.topic = topic_cust_update.name
        pub_ctx_msg_crm2.msg = Message(msg_crm2_value,
                                       mime_type='application/json',
                                       priority=2,
                                       expiration=0.2)

        msg_crm2_id = ps.publish(pub_ctx_msg_crm2).msg.msg_id

        # Billing publishes Msg-Billing1 to /adsl/new
        pub_ctx_msg_billing1 = PubCtx()
        pub_ctx_msg_billing1.client_id = client_id_billing.id
        pub_ctx_msg_billing1.topic = topic_adsl_new.name
        pub_ctx_msg_billing1.msg = Message(msg_billing1_value,
                                           mime_type='application/soap+xml',
                                           priority=3,
                                           expiration=0.3)

        msg_billing1_id = ps.publish(pub_ctx_msg_billing1).msg.msg_id

        # Billing publishes Msg-Billing2 to /adsl/update
        pub_ctx_msg_billing2 = PubCtx()
        pub_ctx_msg_billing2.client_id = client_id_billing.id
        pub_ctx_msg_billing2.topic = topic_adsl_update.name

        # Nothing except payload and expiration set, defaults should be used
        msg_billing2 = Message(msg_billing2_value, expiration=3600)
        pub_ctx_msg_billing2.msg = msg_billing2

        msg_billing2_id = ps.publish(pub_ctx_msg_billing2).msg.msg_id

        keys = self.kvdb.keys('{}*'.format(self.key_prefix))
        eq_(len(keys), 9)

        expected_keys = [
            ps.MSG_VALUES_KEY, ps.MSG_EXPIRE_AT_KEY, ps.LAST_PUB_TIME_KEY
        ]
        for topic in topic_cust_new, topic_cust_update, topic_adsl_new, topic_adsl_update:
            expected_keys.append(ps.MSG_IDS_PREFIX.format(topic.name))

        for key in expected_keys:
            self.assertIn(key, keys)

        # Check values of messages published
        self._check_msg_values_metadata(ps, msg_crm1_id, msg_crm2_id,
                                        msg_billing1_id, msg_billing2_id, True)

        # Now move the messages just published to each of the subscriber's queue.
        # In a real environment this is done by a background job.
        ps.move_to_target_queues()

        # Now all the messages have been moved we can check if everything is in place
        # ready for subscribers to get their messages.

        keys = self.kvdb.keys('{}*'.format(self.key_prefix))
        eq_(len(keys), 9)

        self.assertIn(ps.UNACK_COUNTER_KEY, keys)
        self.assertIn(ps.MSG_VALUES_KEY, keys)

        for sub_key in (sub_key_crm, sub_key_billing, sub_key_erp):
            key = ps.CONSUMER_MSG_IDS_PREFIX.format(sub_key)
            self.assertIn(key, keys)

        self._check_unack_counter(ps, msg_crm1_id, msg_crm2_id,
                                  msg_billing1_id, msg_billing2_id, 1, 1, 2, 2)

        # Check values of messages published are still there
        self._check_msg_values_metadata(ps, msg_crm1_id, msg_crm2_id,
                                        msg_billing1_id, msg_billing2_id, True)

        # Check that each recipient has expected message IDs in its respective message queue
        keys = self.kvdb.keys(ps.CONSUMER_MSG_IDS_PREFIX.format('*'))
        eq_(len(keys), 3)

        msg_ids_crm = self.kvdb.lrange(
            ps.CONSUMER_MSG_IDS_PREFIX.format(sub_key_crm), 0, 100)
        msg_ids_billing = self.kvdb.lrange(
            ps.CONSUMER_MSG_IDS_PREFIX.format(sub_key_billing), 0, 100)
        msg_ids_erp = self.kvdb.lrange(
            ps.CONSUMER_MSG_IDS_PREFIX.format(sub_key_erp), 0, 100)

        eq_(len(msg_ids_crm), 2)
        eq_(len(msg_ids_billing), 2)
        eq_(len(msg_ids_erp), 2)

        self.assertIn(msg_billing1_id, msg_ids_crm)
        self.assertIn(msg_billing2_id, msg_ids_crm)

        self.assertIn(msg_billing1_id, msg_ids_erp)
        self.assertIn(msg_billing2_id, msg_ids_erp)

        self.assertIn(msg_crm1_id, msg_ids_billing)
        self.assertIn(msg_crm1_id, msg_ids_billing)

        # Now that the messages are in queues, let's fetch them

        get_ctx_crm = GetCtx()
        get_ctx_crm.sub_key = sub_key_crm

        get_ctx_billing = GetCtx()
        get_ctx_billing.sub_key = sub_key_billing

        get_ctx_erp = GetCtx()
        get_ctx_erp.sub_key = sub_key_erp

        msgs_crm = sorted(list(ps.get(get_ctx_crm)), key=attrgetter('payload'))
        msgs_billing = sorted(list(ps.get(get_ctx_billing)),
                              key=attrgetter('payload'))
        msgs_erp = sorted(list(ps.get(get_ctx_erp)), key=attrgetter('payload'))

        eq_(len(msgs_crm), 2)
        eq_(len(msgs_billing), 2)
        eq_(len(msgs_erp), 2)

        self._assert_has_msg_metadata(msgs_crm, msg_billing1_id,
                                      'application/soap+xml', 3, 0.3)
        self._assert_has_msg_metadata(msgs_crm, msg_billing2_id, 'text/plain',
                                      5, 3600)

        self._assert_has_msg_metadata(msgs_billing, msg_crm1_id, 'text/xml', 1,
                                      0.1)
        self._assert_has_msg_metadata(msgs_billing, msg_crm2_id,
                                      'application/json', 2, 0.2)

        self._assert_has_msg_metadata(msgs_erp, msg_billing1_id,
                                      'application/soap+xml', 3, 0.3)
        self._assert_has_msg_metadata(msgs_erp, msg_billing2_id, 'text/plain',
                                      5, 3600)

        # Check in-flight status for each message got
        keys = self.kvdb.keys(ps.CONSUMER_IN_FLIGHT_DATA_PREFIX.format('*'))
        eq_(len(keys), 3)

        now = arrow.utcnow()

        self._check_in_flight(ps, now, sub_key_crm, sub_key_billing,
                              sub_key_erp, msg_crm1_id, msg_crm2_id,
                              msg_billing1_id, msg_billing2_id, True, True,
                              True, True, True, True)

        # Messages should still be undelivered hence their unack counters are not touched at this point
        self._check_unack_counter(ps, msg_crm1_id, msg_crm2_id,
                                  msg_billing1_id, msg_billing2_id, 1, 1, 2, 2)

        # ######################################################################################################################

        # Messages are fetched, they can be confirmed or rejected now

        # CRM
        ack_ctx_crm = AckCtx()
        ack_ctx_crm.sub_key = sub_key_crm
        ack_ctx_crm.append(msg_billing1_id)

        reject_ctx_crm = RejectCtx()
        reject_ctx_crm.sub_key = sub_key_crm
        reject_ctx_crm.append(msg_billing2_id)

        ps.acknowledge_delete(ack_ctx_crm)
        ps.reject(reject_ctx_crm)

        # One in-flight less
        self._check_in_flight(ps, now, sub_key_crm, sub_key_billing,
                              sub_key_erp, msg_crm1_id, msg_crm2_id,
                              msg_billing1_id, msg_billing2_id, False, False,
                              True, True, True, True)

        # Rejections, as with reject_ctx_crm, don't change unack count
        self._check_unack_counter(ps, msg_crm1_id, msg_crm2_id,
                                  msg_billing1_id, msg_billing2_id, 1, 1, 1, 2)

        # Billing
        ack_ctx_billing = AckCtx()
        ack_ctx_billing.sub_key = sub_key_billing
        ack_ctx_billing.append(msg_crm1_id)
        ack_ctx_billing.append(msg_crm2_id)

        ps.acknowledge_delete(ack_ctx_billing)

        # Two in-flight less
        self._check_in_flight(ps, now, sub_key_crm, sub_key_billing,
                              sub_key_erp, msg_crm1_id, msg_crm2_id,
                              msg_billing1_id, msg_billing2_id, False, False,
                              False, False, True, True)

        # Again, rejections, as with reject_ctx_crm, don't change unack count
        self._check_unack_counter(ps, None, None, msg_billing1_id,
                                  msg_billing2_id, 1, 1, 1, 2)

        # ERP
        reject_ctx_erp = RejectCtx()
        reject_ctx_erp.sub_key = sub_key_erp
        reject_ctx_erp.append(msg_billing1_id)

        ack_ctx_erp = AckCtx()
        ack_ctx_erp.sub_key = sub_key_erp
        ack_ctx_erp.append(msg_billing2_id)

        ps.reject(reject_ctx_erp)
        ps.acknowledge_delete(ack_ctx_erp)

        # Another in-flight less
        self._check_in_flight(ps, now, sub_key_crm, sub_key_billing,
                              sub_key_erp, msg_crm1_id, msg_crm2_id,
                              msg_billing1_id, msg_billing2_id, False, False,
                              False, False, False, False)

        # And again, rejections, as with reject_ctx_crm, don't change unack count
        self._check_unack_counter(ps, None, None, msg_billing1_id,
                                  msg_billing2_id, 1, 1, 1, 1)

        # Sleep for a moment to make sure enough time passes for messages to expire
        sleep(0.4)

        # Deletes everything except for Msg-Billing2 which has a TTL of 3600
        ps.delete_expired()

        keys = self.kvdb.keys('{}*'.format(self.key_prefix))
        eq_(len(keys), 8)

        expected_keys = [
            ps.MSG_VALUES_KEY, ps.MSG_EXPIRE_AT_KEY, ps.UNACK_COUNTER_KEY,
            ps.CONSUMER_MSG_IDS_PREFIX.format(sub_key_crm)
        ]
        for key in expected_keys:
            self.assertTrue(key in keys,
                            'Key not found `{}` in `{}`'.format(key, keys))

        # Check all the remaining keys - still concerning Msg-Billing2 only
        # because this is the only message that wasn't confirmed nor expired.

        expire_at = self.kvdb.hgetall(ps.MSG_EXPIRE_AT_KEY)
        eq_(len(expire_at), 1)
        eq_(expire_at[msg_billing2_id], msg_billing2.expire_at_utc.isoformat())

        unack_counter = self.kvdb.hgetall(ps.UNACK_COUNTER_KEY)
        eq_(len(unack_counter), 1)
        eq_(unack_counter[msg_billing2_id], '1')

        crm_messages = self.kvdb.lrange(
            ps.CONSUMER_MSG_IDS_PREFIX.format(sub_key_crm), 0, 100)
        eq_(len(crm_messages), 1)
        eq_(crm_messages[0], msg_billing2_id)

        msg_values = self.kvdb.hgetall(ps.MSG_METADATA_KEY)
        eq_(len(msg_values), 2)
        eq_(sorted(loads(msg_values[msg_billing2_id]).items()),
            sorted(msg_billing2.to_dict().items()))
Esempio n. 15
0
    def _after_init_accepted(self, server, locally_deployed):

        # Which components are enabled
        self.component_enabled.stats = asbool(
            self.fs_server_config.component_enabled.stats)
        self.component_enabled.slow_response = asbool(
            self.fs_server_config.component_enabled.slow_response)

        # Pub/sub
        self.pubsub = PubSubAPI(RedisPubSub(self.kvdb.conn))

        # Repo location so that AMQP subprocesses know where to read
        # the server's configuration from.
        self.config.repo_location = self.repo_location

        #
        # Cassandra - start
        #

        query = self.odb.get_cassandra_conn_list(server.cluster.id, True)
        self.config.cassandra_conn = ConfigDict.from_query(
            'cassandra_conn', query)

        query = self.odb.get_cassandra_query_list(server.cluster.id, True)
        self.config.cassandra_query = ConfigDict.from_query(
            'cassandra_query', query)

        #
        # Cassandra - end
        #

        #
        # Search - start
        #

        query = self.odb.get_search_es_list(server.cluster.id, True)
        self.config.search_es = ConfigDict.from_query('search_es', query)

        query = self.odb.get_search_solr_list(server.cluster.id, True)
        self.config.search_solr = ConfigDict.from_query('search_solr', query)

        #
        # Search - end
        #

        #
        # Cloud - start
        #

        # OpenStack - Swift

        query = self.odb.get_cloud_openstack_swift_list(
            server.cluster.id, True)
        self.config.cloud_openstack_swift = ConfigDict.from_query(
            'cloud_openstack_swift', query)

        query = self.odb.get_cloud_aws_s3_list(server.cluster.id, True)
        self.config.cloud_aws_s3 = ConfigDict.from_query('cloud_aws_s3', query)

        #
        # Cloud - end
        #

        # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

        # Services
        query = self.odb.get_service_list(server.cluster.id, True)
        self.config.service = ConfigDict.from_query('service_list', query)

        #
        # Channels - start
        #

        # STOMP
        query = self.odb.get_channel_stomp_list(server.cluster.id, True)
        self.config.channel_stomp = ConfigDict.from_query(
            'channel_stomp', query)

        #
        # Channels - end
        #

        #
        # Outgoing connections - start
        #

        # AMQP
        query = self.odb.get_out_amqp_list(server.cluster.id, True)
        self.config.out_amqp = ConfigDict.from_query('out_amqp', query)

        # FTP
        query = self.odb.get_out_ftp_list(server.cluster.id, True)
        self.config.out_ftp = ConfigDict.from_query('out_ftp', query)

        # JMS WMQ
        query = self.odb.get_out_jms_wmq_list(server.cluster.id, True)
        self.config.out_jms_wmq = ConfigDict.from_query('out_jms_wmq', query)

        # Odoo
        query = self.odb.get_out_odoo_list(server.cluster.id, True)
        self.config.out_odoo = ConfigDict.from_query('out_odoo', query)

        # Plain HTTP
        query = self.odb.get_http_soap_list(server.cluster.id, 'outgoing',
                                            'plain_http', True)
        self.config.out_plain_http = ConfigDict.from_query(
            'out_plain_http', query)

        # SOAP
        query = self.odb.get_http_soap_list(server.cluster.id, 'outgoing',
                                            'soap', True)
        self.config.out_soap = ConfigDict.from_query('out_soap', query)

        # SQL
        query = self.odb.get_out_sql_list(server.cluster.id, True)
        self.config.out_sql = ConfigDict.from_query('out_sql', query)

        # STOMP
        query = self.odb.get_out_stomp_list(server.cluster.id, True)
        self.config.out_stomp = ConfigDict.from_query('out_stomp', query)

        # ZMQ channels
        query = self.odb.get_channel_zmq_list(server.cluster.id, True)
        self.config.channel_zmq = ConfigDict.from_query('channel_zmq', query)

        # ZMQ outgoing
        query = self.odb.get_out_zmq_list(server.cluster.id, True)
        self.config.out_zmq = ConfigDict.from_query('out_zmq', query)

        # WebSocket channels
        query = self.odb.get_channel_web_socket_list(server.cluster.id, True)
        self.config.channel_web_socket = ConfigDict.from_query(
            'channel_web_socket', query)

        #
        # Outgoing connections - end
        #

        # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

        #
        # Notifications - start
        #

        # OpenStack Swift
        query = self.odb.get_notif_cloud_openstack_swift_list(
            server.cluster.id, True)
        self.config.notif_cloud_openstack_swift = ConfigDict.from_query(
            'notif_cloud_openstack_swift', query)

        # SQL
        query = self.odb.get_notif_sql_list(server.cluster.id, True)
        self.config.notif_sql = ConfigDict.from_query('notif_sql', query)

        #
        # Notifications - end
        #

        # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

        #
        # Security - start
        #

        # API keys
        query = self.odb.get_apikey_security_list(server.cluster.id, True)
        self.config.apikey = ConfigDict.from_query('apikey', query)

        # AWS
        query = self.odb.get_aws_security_list(server.cluster.id, True)
        self.config.aws = ConfigDict.from_query('aws', query)

        # HTTP Basic Auth
        query = self.odb.get_basic_auth_list(server.cluster.id, None, True)
        self.config.basic_auth = ConfigDict.from_query('basic_auth', query)

        # HTTP Basic Auth
        query = self.odb.get_jwt_list(server.cluster.id, None, True)
        self.config.jwt = ConfigDict.from_query('jwt', query)

        # NTLM
        query = self.odb.get_ntlm_list(server.cluster.id, True)
        self.config.ntlm = ConfigDict.from_query('ntlm', query)

        # OAuth
        query = self.odb.get_oauth_list(server.cluster.id, True)
        self.config.oauth = ConfigDict.from_query('oauth', query)

        # OpenStack
        query = self.odb.get_openstack_security_list(server.cluster.id, True)
        self.config.openstack_security = ConfigDict.from_query(
            'openstack_security', query)

        # RBAC - permissions
        query = self.odb.get_rbac_permission_list(server.cluster.id, True)
        self.config.rbac_permission = ConfigDict.from_query(
            'rbac_permission', query)

        # RBAC - roles
        query = self.odb.get_rbac_role_list(server.cluster.id, True)
        self.config.rbac_role = ConfigDict.from_query('rbac_role', query)

        # RBAC - client roles
        query = self.odb.get_rbac_client_role_list(server.cluster.id, True)
        self.config.rbac_client_role = ConfigDict.from_query(
            'rbac_client_role', query)

        # RBAC - role permission
        query = self.odb.get_rbac_role_permission_list(server.cluster.id, True)
        self.config.rbac_role_permission = ConfigDict.from_query(
            'rbac_role_permission', query)

        # Technical accounts
        query = self.odb.get_tech_acc_list(server.cluster.id, True)
        self.config.tech_acc = ConfigDict.from_query('tech_acc', query)

        # TLS CA certs
        query = self.odb.get_tls_ca_cert_list(server.cluster.id, True)
        self.config.tls_ca_cert = ConfigDict.from_query('tls_ca_cert', query)

        # TLS channel security
        query = self.odb.get_tls_channel_sec_list(server.cluster.id, True)
        self.config.tls_channel_sec = ConfigDict.from_query(
            'tls_channel_sec', query)

        # TLS key/cert pairs
        query = self.odb.get_tls_key_cert_list(server.cluster.id, True)
        self.config.tls_key_cert = ConfigDict.from_query('tls_key_cert', query)

        # WS-Security
        query = self.odb.get_wss_list(server.cluster.id, True)
        self.config.wss = ConfigDict.from_query('wss', query)

        # XPath
        query = self.odb.get_xpath_sec_list(server.cluster.id, True)
        self.config.xpath_sec = ConfigDict.from_query('xpath_sec', query)

        #
        # Security - end
        #

        # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

        # All the HTTP/SOAP channels.
        http_soap = []
        for item in self.odb.get_http_soap_list(server.cluster.id, 'channel'):

            hs_item = Bunch()
            for key in item.keys():
                hs_item[key] = getattr(item, key)

            hs_item.replace_patterns_json_pointer = item.replace_patterns_json_pointer
            hs_item.replace_patterns_xpath = item.replace_patterns_xpath

            hs_item.match_target = '{}{}{}'.format(hs_item.soap_action,
                                                   MISC.SEPARATOR,
                                                   hs_item.url_path)
            hs_item.match_target_compiled = Matcher(hs_item.match_target)

            http_soap.append(hs_item)

        self.config.http_soap = http_soap

        # Namespaces
        query = self.odb.get_namespace_list(server.cluster.id, True)
        self.config.msg_ns = ConfigDict.from_query('msg_ns', query)

        # XPath
        query = self.odb.get_xpath_list(server.cluster.id, True)
        self.config.xpath = ConfigDict.from_query('msg_xpath', query)

        # JSON Pointer
        query = self.odb.get_json_pointer_list(server.cluster.id, True)
        self.config.json_pointer = ConfigDict.from_query('json_pointer', query)

        # SimpleIO
        self.config.simple_io = ConfigDict('simple_io', Bunch())
        self.config.simple_io['int_parameters'] = self.int_parameters
        self.config.simple_io[
            'int_parameter_suffixes'] = self.int_parameter_suffixes
        self.config.simple_io[
            'bool_parameter_prefixes'] = self.bool_parameter_prefixes

        # Pub/sub config
        self.config.pubsub = Bunch()
        self.config.pubsub.default_consumer = Bunch()
        self.config.pubsub.default_producer = Bunch()

        query = self.odb.get_pubsub_topic_list(server.cluster.id, True)
        self.config.pubsub.topics = ConfigDict.from_query(
            'pubsub_topics', query)

        id, name = self.odb.get_pubsub_default_client(
            server.cluster.id, 'zato.pubsub.default-consumer')
        self.config.pubsub.default_consumer.id, self.config.pubsub.default_consumer.name = id, name

        id, name = self.odb.get_pubsub_default_client(
            server.cluster.id, 'zato.pubsub.default-producer')
        self.config.pubsub.default_producer.id, self.config.pubsub.default_producer.name = id, name

        query = self.odb.get_pubsub_producer_list(server.cluster.id, True)
        self.config.pubsub.producers = ConfigDict.from_query(
            'pubsub_producers', query, list_config=True)

        query = self.odb.get_pubsub_consumer_list(server.cluster.id, True)
        self.config.pubsub.consumers = ConfigDict.from_query(
            'pubsub_consumers', query, list_config=True)

        # E-mail - SMTP
        query = self.odb.get_email_smtp_list(server.cluster.id, True)
        self.config.email_smtp = ConfigDict.from_query('email_smtp', query)

        # E-mail - IMAP
        query = self.odb.get_email_imap_list(server.cluster.id, True)
        self.config.email_imap = ConfigDict.from_query('email_imap', query)

        # Assign config to worker
        self.worker_store.worker_config = self.config
        self.worker_store.pubsub = self.pubsub
        self.worker_store.init()

        # Deployed missing services found on other servers
        if locally_deployed:
            self.deploy_missing_services(locally_deployed)

        # Signal to ODB that we are done with deploying everything
        self.odb.on_deployment_finished()

        # Default content type
        self.json_content_type = self.fs_server_config.content_type.json
        self.plain_xml_content_type = self.fs_server_config.content_type.plain_xml
        self.soap11_content_type = self.fs_server_config.content_type.soap11
        self.soap12_content_type = self.fs_server_config.content_type.soap12