def setUp(self):
super(RedisPubSubTestCase, self).setUp()
self.api = PubSubAPI(RedisPubSub(self.kvdb, self.key_prefix))
class RedisPubSubTestCase(RedisPubSubCommonTestCase):
def setUp(self):
super(RedisPubSubTestCase, self).setUp()
self.api = PubSubAPI(RedisPubSub(self.kvdb, self.key_prefix))
# ################################################################################################################################
def _publish_move(self, move=True, **kwargs):
payload = rand_string()
topic = Topic(rand_string())
self.api.add_topic(topic)
producer = Client(rand_int(), rand_string())
self.api.add_producer(producer, topic)
ctx = self.api.publish(payload,
topic.name,
client_id=producer.id,
**kwargs)
if move:
self.api.impl.move_to_target_queues()
return payload, topic, producer, ctx
def _check_publish(self, **kwargs):
if kwargs:
expected_mime_type = kwargs['mime_type']
expected_priority = kwargs['priority']
expected_expiration = kwargs['expiration']
else:
expected_mime_type = PUB_SUB.DEFAULT_MIME_TYPE
expected_priority = PUB_SUB.DEFAULT_PRIORITY
expected_expiration = PUB_SUB.DEFAULT_EXPIRATION
payload, topic, producer, ctx = self._publish_move(**kwargs)
now = datetime.utcnow()
# ########################################################################################################################
#
# MSG_METADATA_KEY
#
# ########################################################################################################################
msg_metadata_dict = self.kvdb.hgetall(self.api.impl.MSG_METADATA_KEY)
# E.g. {'K0321C8Q5X67N7K2D642ZYZCXY5T': '{"topic": "ab3ee73838d174cd690a1947b56f67674", "priority": 5, "expiration": 60.0,
# "producer": "a951ec619d0f449969529c0bfe8f7900f",
# "creation_time_utc": "2014-04-06T19:51:37.784905", "msg_id": "K0321C8Q5X67N7K2D642ZYZCXY5T",
# "expire_at_utc": "2014-04-06T19:52:37.784905", "mime_type": "text/plain"}'}
self.assertEquals(len(msg_metadata_dict), 1)
self.assertTrue(ctx.msg.msg_id in msg_metadata_dict)
msg_metadata = loads(msg_metadata_dict[ctx.msg.msg_id])
self.assertEquals(msg_metadata['mime_type'], expected_mime_type)
self.assertEquals(msg_metadata['priority'], expected_priority)
self.assertEquals(msg_metadata['expiration'], expected_expiration)
self.assertEquals(msg_metadata['topic'], topic.name)
self.assertEquals(msg_metadata['producer'], producer.name)
creation_time_utc = parse(msg_metadata['creation_time_utc'])
expire_at_utc = parse(msg_metadata['expire_at_utc'])
self.assertTrue(
creation_time_utc < now,
'creation_time_utc:`{}` is not less than now:`{}`'.format(
creation_time_utc, now))
self.assertTrue(
expire_at_utc > now,
'creation_time_utc:`{}` is not greater than now:`{}`'.format(
expire_at_utc, now))
# ########################################################################################################################
#
# LAST_PUB_TIME_KEY
#
# ########################################################################################################################
last_pub_time = self.kvdb.hgetall(self.api.impl.LAST_PUB_TIME_KEY)
self.assertEquals(len(last_pub_time), 1)
last_pub_time = parse(last_pub_time[topic.name])
self.assertTrue(
last_pub_time < now,
'last_pub_time:`{}` is not less than now:`{}`'.format(
last_pub_time, now))
# ########################################################################################################################
#
# MSG_EXPIRE_AT_KEY
#
# ########################################################################################################################
msg_expire_at = self.kvdb.hgetall(self.api.impl.MSG_EXPIRE_AT_KEY)
self.assertEquals(len(msg_expire_at), 1)
msg_expire_at = parse(msg_expire_at[ctx.msg.msg_id])
self.assertTrue(
msg_expire_at > now,
'msg_expire_at:`{}` is not greater than now:`{}`'.format(
msg_expire_at, now))
# ########################################################################################################################
#
# LAST_SEEN_PRODUCER_KEY
#
# ########################################################################################################################
last_seen_producer = self.kvdb.hgetall(
self.api.impl.LAST_SEEN_PRODUCER_KEY)
self.assertEquals(len(last_seen_producer), 1)
last_seen_producer = parse(last_seen_producer[str(producer.id)])
self.assertTrue(
last_seen_producer < now,
'last_seen_producer:`{}` is not less than now:`{}`'.format(
last_seen_producer, now))
# ########################################################################################################################
#
# MSG_VALUES_KEY
#
# ########################################################################################################################
msg_values = self.kvdb.hgetall(self.api.impl.MSG_VALUES_KEY)
self.assertEquals(len(msg_values), 1)
self.assertEquals(payload, msg_values[ctx.msg.msg_id])
def test_publish_defaults(self):
self._check_publish()
def test_publish_custom_attrs(self):
self._check_publish(
**{
'mime_type': rand_string(),
'priority': rand_int(),
'expiration': rand_int(1000, 2000),
'msg_id': rand_string(),
})
# ################################################################################################################################
def test_delete_metadata(self):
payload, topic, producer, ctx = self._publish_move(move=False)
consumer = Consumer(rand_int(), rand_string())
self.api.add_consumer(consumer, topic)
sub_key = self.api.subscribe(consumer.id, topic.name)
self.api.impl.move_to_target_queues()
self._check_consumer_queue_before_get(ctx, sub_key)
self._check_get(ctx, sub_key, topic, producer, consumer)
self.api.acknowledge(sub_key, ctx.msg.msg_id)
# Ok, we should now have metadata for the consumer, producer and topic.
last_seen_consumer = self.api.impl.kvdb.hkeys(
self.api.impl.LAST_SEEN_CONSUMER_KEY)
last_seen_producer = self.api.impl.kvdb.hkeys(
self.api.impl.LAST_SEEN_PRODUCER_KEY)
last_pub_time = self.api.impl.kvdb.hkeys(
self.api.impl.LAST_PUB_TIME_KEY)
self.assertIn(str(consumer.id), last_seen_consumer)
self.assertIn(str(producer.id), last_seen_producer)
self.assertIn(topic.name, last_pub_time)
self.api.impl.delete_producer(producer, topic)
last_seen_producer = self.api.impl.kvdb.hkeys(
self.api.impl.LAST_SEEN_PRODUCER_KEY)
self.assertNotIn(str(producer.id), last_seen_producer)
self.api.impl.delete_consumer(consumer, topic)
last_seen_consumer = self.api.impl.kvdb.hkeys(
self.api.impl.LAST_SEEN_CONSUMER_KEY)
self.assertNotIn(str(consumer.id), last_seen_consumer)
self.api.impl.delete_topic(topic)
last_pub_time = self.api.impl.kvdb.hkeys(
self.api.impl.LAST_PUB_TIME_KEY)
self.assertNotIn(topic.name, last_pub_time)
# ################################################################################################################################
def test_subscribe(self):
client_id, client_name = rand_int(), rand_string()
client = Client(client_id, client_name)
topics = rand_string(rand_int())
sub_key = self.api.subscribe(client.id, topics)
self.assertEquals(self.api.impl.sub_to_cons[sub_key], client_id)
self.assertEquals(self.api.impl.cons_to_sub[client_id], sub_key)
self.assertEquals(sorted(self.api.impl.cons_to_topic[client_id]),
sorted(topics))
for topic in topics:
self.assertIn(client_id, self.api.impl.topic_to_cons[topic])
# ################################################################################################################################
def _check_consumer_queue_before_get(self, ctx, sub_key):
# ########################################################################################################################
#
# UNACK_COUNTER_KEY
#
# ########################################################################################################################
unack_counter = self.kvdb.hgetall(self.api.impl.UNACK_COUNTER_KEY)
self.assertEquals(len(unack_counter), 1)
self.assertEqual(unack_counter[ctx.msg.msg_id],
'1') # One subscriber hence one undelivered message
# ########################################################################################################################
#
# CONSUMER_MSG_IDS_PREFIX
#
# ########################################################################################################################
consumer_msg_ids = self.kvdb.lrange(
self.api.impl.CONSUMER_MSG_IDS_PREFIX.format(sub_key), 0, -1)
self.assertEquals(consumer_msg_ids, [ctx.msg.msg_id])
def _check_get(self, ctx, sub_key, topic, producer, client):
msg = list(self.api.get(sub_key))[0].to_dict()
self.assertEquals(msg['topic'], topic.name)
self.assertEquals(msg['priority'], PUB_SUB.DEFAULT_PRIORITY)
self.assertEquals(msg['expiration'], PUB_SUB.DEFAULT_EXPIRATION)
self.assertEquals(msg['producer'], producer.name)
self.assertEquals(msg['msg_id'], ctx.msg.msg_id)
self.assertEquals(msg['mime_type'], PUB_SUB.DEFAULT_MIME_TYPE)
now = datetime.utcnow()
creation_time_utc = parse(msg['creation_time_utc'])
expire_at_utc = parse(msg['expire_at_utc'])
self.assertTrue(
creation_time_utc < now,
'creation_time_utc:`{}` is not less than now:`{}`'.format(
creation_time_utc, now))
self.assertTrue(
expire_at_utc > now,
'creation_time_utc:`{}` is not greater than now:`{}`'.format(
expire_at_utc, now))
# ########################################################################################################################
#
# LAST_SEEN_CONSUMER_KEY
#
# ########################################################################################################################
last_seen_consumer = self.kvdb.hgetall(
self.api.impl.LAST_SEEN_CONSUMER_KEY)
self.assertEquals(len(last_seen_consumer), 1)
last_seen_consumer = parse(last_seen_consumer[str(client.id)])
self.assertTrue(
last_seen_consumer < now,
'last_seen_consumer:`{}` is not less than now:`{}`'.format(
last_seen_consumer, now))
# ########################################################################################################################
#
# CONSUMER_IN_FLIGHT_IDS_PREFIX
#
# ########################################################################################################################
consumer_id_flight_ids = self.kvdb.smembers(
self.api.impl.CONSUMER_IN_FLIGHT_IDS_PREFIX.format(sub_key))
self.assertEquals(len(consumer_id_flight_ids), 1)
self.assertEqual(list(consumer_id_flight_ids), [ctx.msg.msg_id])
# ########################################################################################################################
#
# CONSUMER_IN_FLIGHT_DATA_PREFIX
#
# ########################################################################################################################
consumer_in_flight_data = self.kvdb.hgetall(
self.api.impl.CONSUMER_IN_FLIGHT_DATA_PREFIX.format(sub_key))
self.assertEquals(len(consumer_in_flight_data), 1)
consumer_in_flight_data = parse(
consumer_in_flight_data[ctx.msg.msg_id])
self.assertTrue(
consumer_in_flight_data < now,
'consumer_in_flight_data:`{}` is not less than now:`{}`'.format(
consumer_in_flight_data, now))
# There should still be one unacknowledged message.
unack_counter = self.kvdb.hgetall(self.api.impl.UNACK_COUNTER_KEY)
self.assertEquals(len(unack_counter), 1)
self.assertEqual(unack_counter[ctx.msg.msg_id],
'1') # One subscriber hence one undelivered message
def test_get_reject_acknowledge(self):
payload, topic, producer, ctx = self._publish_move(move=False)
client_id, client_name = rand_int(), rand_string()
client = Client(client_id, client_name)
sub_key = self.api.subscribe(client.id, topic.name)
# Moves a message to the consumer's queue
self.api.impl.move_to_target_queues()
self._check_consumer_queue_before_get(ctx, sub_key)
# Consumer gets a message which puts it in the in-flight state.
self._check_get(ctx, sub_key, topic, producer, client)
# However, there should be nothing in the consumer's queue.
consumer_msg_ids = self.kvdb.lrange(
self.api.impl.CONSUMER_MSG_IDS_PREFIX.format(sub_key), 0, -1)
self.assertEquals(consumer_msg_ids, [])
# Consumer rejects the message which puts it back on a queue.
self.api.reject(sub_key, ctx.msg.msg_id)
# After rejection it's as though the message has just been published.
self._check_consumer_queue_before_get(ctx, sub_key)
# Get after rejection works as before.
self._check_get(ctx, sub_key, topic, producer, client)
# Consumer acknowledges a message.
self.api.acknowledge(sub_key, ctx.msg.msg_id)
# This was the only one subscription so now that the message has been delivered
# there should be no trace of it in backend.
# The only keys left are LAST_PUB_TIME_KEY, LAST_SEEN_CONSUMER_KEY and LAST_SEEN_PRODUCER_KEY - nothing else.
keys = self.kvdb.keys('{}*'.format(self.key_prefix))
self.assertEquals(len(keys), 3)
now = datetime.utcnow()
last_pub_time = parse(
self.kvdb.hgetall(self.api.impl.LAST_PUB_TIME_KEY)[topic.name])
last_seen_consumer = parse(
self.kvdb.hgetall(self.api.impl.LAST_SEEN_CONSUMER_KEY)[str(
client.id)])
last_seen_producer = parse(
self.kvdb.hgetall(self.api.impl.LAST_SEEN_PRODUCER_KEY)[str(
producer.id)])
self.assertTrue(
last_pub_time < now,
'last_pub_time:`{}` is not less than now:`{}`'.format(
last_pub_time, now))
self.assertTrue(
last_seen_consumer < now,
'last_seen_consumer:`{}` is not less than now:`{}`'.format(
last_seen_consumer, now))
self.assertTrue(
last_seen_producer < now,
'last_seen_producer:`{}` is not less than now:`{}`'.format(
last_seen_producer, now))
# ################################################################################################################################
def test_pub_sub_exception(self):
invalid_sub_key = rand_string()
valid_sub_key = rand_string()
client_id = rand_int()
topic_name = rand_string()
consumer = Consumer(client_id, rand_string(), sub_key=valid_sub_key)
topic = Topic(topic_name)
# Without adding consumer key, validation won't succeed.
self.assertRaises(PubSubException, self.api.impl.validate_sub_key,
invalid_sub_key)
self.assertRaises(PubSubException, self.api.impl.validate_sub_key,
valid_sub_key)
# After adding a subscription key no error should be raised.
self.api.impl.add_consumer(consumer, topic)
self.api.impl.add_subscription(valid_sub_key, client_id, topic_name)
self.assertRaises(PubSubException, self.api.impl.validate_sub_key,
invalid_sub_key)
self.api.impl.validate_sub_key(
valid_sub_key) # Should not raise any exception now.
self.api.impl.delete_consumer(consumer, topic)
# After deleting a consumer, validation won't succeed anymore.
self.assertRaises(PubSubException, self.api.impl.validate_sub_key,
invalid_sub_key)
self.assertRaises(PubSubException, self.api.impl.validate_sub_key,
valid_sub_key)
def invoke_func_sub_key(func, sub_key, *args):
list(func(sub_key, *args))
self.assertRaises(PubSubException, invoke_func_sub_key, self.api.get,
valid_sub_key)
self.assertRaises(PubSubException, invoke_func_sub_key, self.api.get,
invalid_sub_key)
self.assertRaises(PubSubException, invoke_func_sub_key,
self.api.acknowledge, valid_sub_key, 'abc')
self.assertRaises(PubSubException, invoke_func_sub_key,
self.api.acknowledge, invalid_sub_key, 'def')
self.assertRaises(PubSubException, invoke_func_sub_key,
self.api.reject, valid_sub_key, 'abc')
self.assertRaises(PubSubException, invoke_func_sub_key,
self.api.reject, invalid_sub_key, 'def')
def test_publish_exceptions(self):
payload = rand_string()
producer = Client(rand_int(), rand_string())
def invoke_publish(payload, topic, producer_id):
self.api.publish(payload, topic, client_id=producer_id)
# KeyError because no such producer is in self.api.impl.producers.
self.assertRaises(KeyError, invoke_publish, payload, rand_string(),
producer.id)
# Adding a producer but still, no such topic.
self.api.add_producer(producer, Topic(rand_string()))
self.assertRaises(PubSubException, invoke_publish, payload,
rand_string(), producer.id)
# Adding a topic but still PubSubException is raised because the producer is not allowed to use it.
topic = Topic(rand_string())
self.api.add_topic(topic)
self.assertRaises(PubSubException, invoke_publish, payload, topic.name,
producer.id)
# Combining the topic and producer, no exception is raised now.
self.api.add_producer(producer, topic)
invoke_publish(payload, topic.name, producer.id)
# But it's not possible to publish to inactive topics.
self.api.impl.topics[topic.name].is_active = False
self.assertRaises(PubSubException, invoke_publish, payload, topic.name,
producer.id)
# Make the topic active and it can be published to again.
self.api.impl.topics[topic.name].is_active = True
invoke_publish(payload, topic.name, producer.id)
# Inactive producers cannot publish to topics either.
self.api.impl.producers[producer.id].is_active = False
self.assertRaises(PubSubException, invoke_publish, payload, topic.name,
producer.id)
# Making them active means they can publish again.
self.api.impl.producers[producer.id].is_active = True
invoke_publish(payload, topic.name, producer.id)
def test_ping(self):
response = self.api.impl.ping()
self.assertIsInstance(response, bool)
self.assertEquals(response, True)
# ################################################################################################################################
def test_default_clients(self):
# Initially, default clients are dummy ones.
default_consumer = self.api.get_default_consumer()
default_producer = self.api.get_default_producer()
self.assertEquals(default_consumer.id, None)
self.assertEquals(default_consumer.name, None)
self.assertEquals(default_consumer.is_active, True)
self.assertEquals(default_producer.id, None)
self.assertEquals(default_producer.name, None)
self.assertEquals(default_producer.is_active, True)
cons_id = rand_int()
cons_name = rand_string()
cons_is_active = rand_bool()
prod_name = rand_string()
prod_id = rand_int()
prod_is_active = rand_bool()
cons = Client(cons_id, cons_name, cons_is_active)
prod = Client(prod_id, prod_name, prod_is_active)
self.api.set_default_consumer(cons)
self.api.set_default_producer(prod)
default_consumer = self.api.get_default_consumer()
default_producer = self.api.get_default_producer()
self.assertEquals(default_consumer.id, cons_id)
self.assertEquals(default_consumer.name, cons_name)
self.assertEquals(default_consumer.is_active, cons_is_active)
self.assertEquals(default_producer.id, prod_id)
self.assertEquals(default_producer.name, prod_name)
self.assertEquals(default_producer.is_active, prod_is_active)
# ################################################################################################################################
def test_topic_add(self):
name = rand_string()
is_active = rand_bool()
is_fifo = rand_bool()
max_depth = rand_int()
topic = Topic(name, is_active, is_fifo, max_depth)
self.api.add_topic(topic)
self.assertIn(name, self.api.impl.topics)
self.assertEquals(len(self.api.impl.topics), 1)
given = self.api.impl.topics[name]
self.assertEquals(given.name, name)
self.assertEquals(given.is_active, is_active)
self.assertEquals(given.is_fifo, is_fifo)
self.assertEquals(given.max_depth, max_depth)
# Adding topic of the same name should not create a new topic because impl.topics is a dictionary
self.api.add_topic(topic)
self.assertEquals(len(self.api.impl.topics), 1)
def test_topic_update(self):
self.test_topic_add(
) # updating a topic works the same like creating it
def setUp(self):
super(RedisPubSubTestCase, self).setUp()
self.api = PubSubAPI(RedisPubSub(self.kvdb, self.key_prefix))
class RedisPubSubTestCase(RedisPubSubCommonTestCase):
def setUp(self):
super(RedisPubSubTestCase, self).setUp()
self.api = PubSubAPI(RedisPubSub(self.kvdb, self.key_prefix))
# ################################################################################################################################
def _publish_move(self, move=True, **kwargs):
payload = rand_string()
topic = Topic(rand_string())
self.api.add_topic(topic)
producer = Client(rand_int(), rand_string())
self.api.add_producer(producer, topic)
ctx = self.api.publish(payload, topic.name, client_id=producer.id, **kwargs)
if move:
self.api.impl.move_to_target_queues()
return payload, topic, producer, ctx
def _check_publish(self, **kwargs):
if kwargs:
expected_mime_type = kwargs['mime_type']
expected_priority = kwargs['priority']
expected_expiration = kwargs['expiration']
expected_msg_id = kwargs['msg_id']
else:
expected_mime_type = PUB_SUB.DEFAULT_MIME_TYPE
expected_priority = PUB_SUB.DEFAULT_PRIORITY
expected_expiration = PUB_SUB.DEFAULT_EXPIRATION
expected_msg_id = None
payload, topic, producer, ctx = self._publish_move(**kwargs)
now = datetime.utcnow()
# ########################################################################################################################
#
# MSG_METADATA_KEY
#
# ########################################################################################################################
msg_metadata_dict = self.kvdb.hgetall(self.api.impl.MSG_METADATA_KEY)
# E.g. {'K0321C8Q5X67N7K2D642ZYZCXY5T': '{"topic": "ab3ee73838d174cd690a1947b56f67674", "priority": 5, "expiration": 60.0,
# "producer": "a951ec619d0f449969529c0bfe8f7900f",
# "creation_time_utc": "2014-04-06T19:51:37.784905", "msg_id": "K0321C8Q5X67N7K2D642ZYZCXY5T",
# "expire_at_utc": "2014-04-06T19:52:37.784905", "mime_type": "text/plain"}'}
self.assertEquals(len(msg_metadata_dict), 1)
self.assertTrue(ctx.msg.msg_id in msg_metadata_dict)
msg_metadata = loads(msg_metadata_dict[ctx.msg.msg_id])
self.assertEquals(msg_metadata['mime_type'], expected_mime_type)
self.assertEquals(msg_metadata['priority'], expected_priority)
self.assertEquals(msg_metadata['expiration'], expected_expiration)
self.assertEquals(msg_metadata['topic'], topic.name)
self.assertEquals(msg_metadata['producer'], producer.name)
creation_time_utc = parse(msg_metadata['creation_time_utc'])
expire_at_utc = parse(msg_metadata['expire_at_utc'])
self.assertTrue(creation_time_utc < now, 'creation_time_utc:`{}` is not less than now:`{}`'.format(creation_time_utc, now))
self.assertTrue(expire_at_utc > now, 'creation_time_utc:`{}` is not greater than now:`{}`'.format(expire_at_utc, now))
# ########################################################################################################################
#
# LAST_PUB_TIME_KEY
#
# ########################################################################################################################
last_pub_time = self.kvdb.hgetall(self.api.impl.LAST_PUB_TIME_KEY)
self.assertEquals(len(last_pub_time), 1)
last_pub_time = parse(last_pub_time[topic.name])
self.assertTrue(last_pub_time < now, 'last_pub_time:`{}` is not less than now:`{}`'.format(last_pub_time, now))
# ########################################################################################################################
#
# MSG_EXPIRE_AT_KEY
#
# ########################################################################################################################
msg_expire_at = self.kvdb.hgetall(self.api.impl.MSG_EXPIRE_AT_KEY)
self.assertEquals(len(msg_expire_at), 1)
msg_expire_at = parse(msg_expire_at[ctx.msg.msg_id])
self.assertTrue(msg_expire_at > now, 'msg_expire_at:`{}` is not greater than now:`{}`'.format(msg_expire_at, now))
# ########################################################################################################################
#
# LAST_SEEN_PRODUCER_KEY
#
# ########################################################################################################################
last_seen_producer = self.kvdb.hgetall(self.api.impl.LAST_SEEN_PRODUCER_KEY)
self.assertEquals(len(last_seen_producer), 1)
last_seen_producer = parse(last_seen_producer[str(producer.id)])
self.assertTrue(last_seen_producer < now, 'last_seen_producer:`{}` is not less than now:`{}`'.format(last_seen_producer, now))
# ########################################################################################################################
#
# MSG_VALUES_KEY
#
# ########################################################################################################################
msg_values = self.kvdb.hgetall(self.api.impl.MSG_VALUES_KEY)
self.assertEquals(len(msg_values), 1)
self.assertEquals(payload, msg_values[ctx.msg.msg_id])
def test_publish_defaults(self):
self._check_publish()
def test_publish_custom_attrs(self):
self._check_publish(**{
'mime_type': rand_string(),
'priority': rand_int(),
'expiration': rand_int(1000, 2000),
'msg_id': rand_string(),
})
# ################################################################################################################################
def test_delete_metadata(self):
payload, topic, producer, ctx = self._publish_move(move=False)
consumer = Consumer(rand_int(), rand_string())
self.api.add_consumer(consumer, topic)
sub_key = self.api.subscribe(consumer.id, topic.name)
self.api.impl.move_to_target_queues()
self._check_consumer_queue_before_get(ctx, sub_key)
self._check_get(ctx, sub_key, topic, producer, consumer)
self.api.acknowledge(sub_key, ctx.msg.msg_id)
# Ok, we should now have metadata for the consumer, producer and topic.
last_seen_consumer = self.api.impl.kvdb.hkeys(self.api.impl.LAST_SEEN_CONSUMER_KEY)
last_seen_producer = self.api.impl.kvdb.hkeys(self.api.impl.LAST_SEEN_PRODUCER_KEY)
last_pub_time = self.api.impl.kvdb.hkeys(self.api.impl.LAST_PUB_TIME_KEY)
self.assertIn(str(consumer.id), last_seen_consumer)
self.assertIn(str(producer.id), last_seen_producer)
self.assertIn(topic.name, last_pub_time)
self.api.impl.delete_producer(producer, topic)
last_seen_producer = self.api.impl.kvdb.hkeys(self.api.impl.LAST_SEEN_PRODUCER_KEY)
self.assertNotIn(str(producer.id), last_seen_producer)
self.api.impl.delete_consumer(consumer, topic)
last_seen_consumer = self.api.impl.kvdb.hkeys(self.api.impl.LAST_SEEN_CONSUMER_KEY)
self.assertNotIn(str(consumer.id), last_seen_consumer)
self.api.impl.delete_topic(topic)
last_pub_time = self.api.impl.kvdb.hkeys(self.api.impl.LAST_PUB_TIME_KEY)
self.assertNotIn(topic.name, last_pub_time)
# ################################################################################################################################
def test_subscribe(self):
client_id, client_name = rand_int(), rand_string()
client = Client(client_id, client_name)
topics = rand_string(rand_int())
sub_key = self.api.subscribe(client.id, topics)
self.assertEquals(self.api.impl.sub_to_cons[sub_key], client_id)
self.assertEquals(self.api.impl.cons_to_sub[client_id], sub_key)
self.assertEquals(sorted(self.api.impl.cons_to_topic[client_id]), sorted(topics))
for topic in topics:
self.assertIn(client_id, self.api.impl.topic_to_cons[topic])
# ################################################################################################################################
def _check_consumer_queue_before_get(self, ctx, sub_key):
# ########################################################################################################################
#
# UNACK_COUNTER_KEY
#
# ########################################################################################################################
unack_counter = self.kvdb.hgetall(self.api.impl.UNACK_COUNTER_KEY)
self.assertEquals(len(unack_counter), 1)
self.assertEqual(unack_counter[ctx.msg.msg_id], '1') # One subscriber hence one undelivered message
# ########################################################################################################################
#
# CONSUMER_MSG_IDS_PREFIX
#
# ########################################################################################################################
consumer_msg_ids = self.kvdb.lrange(self.api.impl.CONSUMER_MSG_IDS_PREFIX.format(sub_key), 0, -1)
self.assertEquals(consumer_msg_ids, [ctx.msg.msg_id])
def _check_get(self, ctx, sub_key, topic, producer, client):
msg = list(self.api.get(sub_key))[0].to_dict()
self.assertEquals(msg['topic'], topic.name)
self.assertEquals(msg['priority'], PUB_SUB.DEFAULT_PRIORITY)
self.assertEquals(msg['expiration'], PUB_SUB.DEFAULT_EXPIRATION)
self.assertEquals(msg['producer'], producer.name)
self.assertEquals(msg['msg_id'], ctx.msg.msg_id)
self.assertEquals(msg['mime_type'], PUB_SUB.DEFAULT_MIME_TYPE)
now = datetime.utcnow()
creation_time_utc = parse(msg['creation_time_utc'])
expire_at_utc = parse(msg['expire_at_utc'])
self.assertTrue(creation_time_utc < now, 'creation_time_utc:`{}` is not less than now:`{}`'.format(creation_time_utc, now))
self.assertTrue(expire_at_utc > now, 'creation_time_utc:`{}` is not greater than now:`{}`'.format(expire_at_utc, now))
# ########################################################################################################################
#
# LAST_SEEN_CONSUMER_KEY
#
# ########################################################################################################################
last_seen_consumer = self.kvdb.hgetall(self.api.impl.LAST_SEEN_CONSUMER_KEY)
self.assertEquals(len(last_seen_consumer), 1)
last_seen_consumer = parse(last_seen_consumer[str(client.id)])
self.assertTrue(last_seen_consumer < now, 'last_seen_consumer:`{}` is not less than now:`{}`'.format(last_seen_consumer, now))
# ########################################################################################################################
#
# CONSUMER_IN_FLIGHT_IDS_PREFIX
#
# ########################################################################################################################
consumer_id_flight_ids = self.kvdb.smembers(self.api.impl.CONSUMER_IN_FLIGHT_IDS_PREFIX.format(sub_key))
self.assertEquals(len(consumer_id_flight_ids), 1)
self.assertEqual(list(consumer_id_flight_ids), [ctx.msg.msg_id])
# ########################################################################################################################
#
# CONSUMER_IN_FLIGHT_DATA_PREFIX
#
# ########################################################################################################################
consumer_in_flight_data = self.kvdb.hgetall(self.api.impl.CONSUMER_IN_FLIGHT_DATA_PREFIX.format(sub_key))
self.assertEquals(len(consumer_in_flight_data), 1)
consumer_in_flight_data = parse(consumer_in_flight_data[ctx.msg.msg_id])
self.assertTrue(
consumer_in_flight_data < now, 'consumer_in_flight_data:`{}` is not less than now:`{}`'.format(
consumer_in_flight_data, now))
# There should still be one unacknowledged message.
unack_counter = self.kvdb.hgetall(self.api.impl.UNACK_COUNTER_KEY)
self.assertEquals(len(unack_counter), 1)
self.assertEqual(unack_counter[ctx.msg.msg_id], '1') # One subscriber hence one undelivered message
def test_get_reject_acknowledge(self):
payload, topic, producer, ctx = self._publish_move(move=False)
client_id, client_name = rand_int(), rand_string()
client = Client(client_id, client_name)
sub_key = self.api.subscribe(client.id, topic.name)
# Moves a message to the consumer's queue
self.api.impl.move_to_target_queues()
self._check_consumer_queue_before_get(ctx, sub_key)
# Consumer gets a message which puts it in the in-flight state.
self._check_get(ctx, sub_key, topic, producer, client)
# However, there should be nothing in the consumer's queue.
consumer_msg_ids = self.kvdb.lrange(self.api.impl.CONSUMER_MSG_IDS_PREFIX.format(sub_key), 0, -1)
self.assertEquals(consumer_msg_ids, [])
# Consumer rejects the message which puts it back on a queue.
self.api.reject(sub_key, ctx.msg.msg_id)
# After rejection it's as though the message has just been published.
self._check_consumer_queue_before_get(ctx, sub_key)
# Get after rejection works as before.
self._check_get(ctx, sub_key, topic, producer, client)
# Consumer acknowledges a message.
self.api.acknowledge(sub_key, ctx.msg.msg_id)
# This was the only one subscription so now that the message has been delivered
# there should be no trace of it in backend.
# The only keys left are LAST_PUB_TIME_KEY, LAST_SEEN_CONSUMER_KEY and LAST_SEEN_PRODUCER_KEY - nothing else.
keys = self.kvdb.keys('{}*'.format(self.key_prefix))
self.assertEquals(len(keys), 3)
now = datetime.utcnow()
last_pub_time = parse(self.kvdb.hgetall(self.api.impl.LAST_PUB_TIME_KEY)[topic.name])
last_seen_consumer = parse(self.kvdb.hgetall(self.api.impl.LAST_SEEN_CONSUMER_KEY)[str(client.id)])
last_seen_producer = parse(self.kvdb.hgetall(self.api.impl.LAST_SEEN_PRODUCER_KEY)[str(producer.id)])
self.assertTrue(last_pub_time < now, 'last_pub_time:`{}` is not less than now:`{}`'.format(last_pub_time, now))
self.assertTrue(last_seen_consumer < now, 'last_seen_consumer:`{}` is not less than now:`{}`'.format(last_seen_consumer, now))
self.assertTrue(last_seen_producer < now, 'last_seen_producer:`{}` is not less than now:`{}`'.format(last_seen_producer, now))
# ################################################################################################################################
def test_pub_sub_exception(self):
invalid_sub_key = rand_string()
valid_sub_key = rand_string()
client_id = rand_int()
topic_name = rand_string()
consumer = Consumer(client_id, rand_string(), sub_key=valid_sub_key)
topic = Topic(topic_name)
# Without adding consumer key, validation won't succeed.
self.assertRaises(PubSubException, self.api.impl.validate_sub_key, invalid_sub_key)
self.assertRaises(PubSubException, self.api.impl.validate_sub_key, valid_sub_key)
# After adding a subscription key no error should be raised.
self.api.impl.add_consumer(consumer, topic)
self.api.impl.add_subscription(valid_sub_key, client_id, topic_name)
self.assertRaises(PubSubException, self.api.impl.validate_sub_key, invalid_sub_key)
self.api.impl.validate_sub_key(valid_sub_key) # Should not raise any exception now.
self.api.impl.delete_consumer(consumer, topic)
# After deleting a consumer, validation won't succeed anymore.
self.assertRaises(PubSubException, self.api.impl.validate_sub_key, invalid_sub_key)
self.assertRaises(PubSubException, self.api.impl.validate_sub_key, valid_sub_key)
def invoke_func_sub_key(func, sub_key, *args):
list(func(sub_key, *args))
self.assertRaises(PubSubException, invoke_func_sub_key, self.api.get, valid_sub_key)
self.assertRaises(PubSubException, invoke_func_sub_key, self.api.get, invalid_sub_key)
self.assertRaises(PubSubException, invoke_func_sub_key, self.api.acknowledge, valid_sub_key, 'abc')
self.assertRaises(PubSubException, invoke_func_sub_key, self.api.acknowledge, invalid_sub_key, 'def')
self.assertRaises(PubSubException, invoke_func_sub_key, self.api.reject, valid_sub_key, 'abc')
self.assertRaises(PubSubException, invoke_func_sub_key, self.api.reject, invalid_sub_key, 'def')
def test_publish_exceptions(self):
payload = rand_string()
producer = Client(rand_int(), rand_string())
def invoke_publish(payload, topic, producer_id):
self.api.publish(payload, topic, client_id=producer_id)
# KeyError because no such producer is in self.api.impl.producers.
self.assertRaises(KeyError, invoke_publish, payload, rand_string(), producer.id)
# Adding a producer but still, no such topic.
self.api.add_producer(producer, Topic(rand_string()))
self.assertRaises(PubSubException, invoke_publish, payload, rand_string(), producer.id)
# Adding a topic but still PubSubException is raised because the producer is not allowed to use it.
topic = Topic(rand_string())
self.api.add_topic(topic)
self.assertRaises(PubSubException, invoke_publish, payload, topic.name, producer.id)
# Combining the topic and producer, no exception is raised now.
self.api.add_producer(producer, topic)
invoke_publish(payload, topic.name, producer.id)
# But it's not possible to publish to inactive topics.
self.api.impl.topics[topic.name].is_active = False
self.assertRaises(PubSubException, invoke_publish, payload, topic.name, producer.id)
# Make the topic active and it can be published to again.
self.api.impl.topics[topic.name].is_active = True
invoke_publish(payload, topic.name, producer.id)
# Inactive producers cannot publish to topics either.
self.api.impl.producers[producer.id].is_active = False
self.assertRaises(PubSubException, invoke_publish, payload, topic.name, producer.id)
# Making them active means they can publish again.
self.api.impl.producers[producer.id].is_active = True
invoke_publish(payload, topic.name, producer.id)
def test_ping(self):
response = self.api.impl.ping()
self.assertIsInstance(response, bool)
self.assertEquals(response, True)
# ################################################################################################################################
def test_default_clients(self):
# Initially, default clients are dummy ones.
default_consumer = self.api.get_default_consumer()
default_producer = self.api.get_default_producer()
self.assertEquals(default_consumer.id, None)
self.assertEquals(default_consumer.name, None)
self.assertEquals(default_consumer.is_active, True)
self.assertEquals(default_producer.id, None)
self.assertEquals(default_producer.name, None)
self.assertEquals(default_producer.is_active, True)
cons_id = rand_int()
cons_name = rand_string()
cons_is_active = rand_bool()
prod_name = rand_string()
prod_id = rand_int()
prod_is_active = rand_bool()
cons = Client(cons_id, cons_name, cons_is_active)
prod = Client(prod_id, prod_name, prod_is_active)
self.api.set_default_consumer(cons)
self.api.set_default_producer(prod)
default_consumer = self.api.get_default_consumer()
default_producer = self.api.get_default_producer()
self.assertEquals(default_consumer.id, cons_id)
self.assertEquals(default_consumer.name, cons_name)
self.assertEquals(default_consumer.is_active, cons_is_active)
self.assertEquals(default_producer.id, prod_id)
self.assertEquals(default_producer.name, prod_name)
self.assertEquals(default_producer.is_active, prod_is_active)
# ################################################################################################################################
def test_topic_add(self):
name = rand_string()
is_active = rand_bool()
is_fifo = rand_bool()
max_depth = rand_int()
topic = Topic(name, is_active, is_fifo, max_depth)
self.api.add_topic(topic)
self.assertIn(name, self.api.impl.topics)
self.assertEquals(len(self.api.impl.topics), 1)
given = self.api.impl.topics[name]
self.assertEquals(given.name, name)
self.assertEquals(given.is_active, is_active)
self.assertEquals(given.is_fifo, is_fifo)
self.assertEquals(given.max_depth, max_depth)
# Adding topic of the same name should not create a new topic because impl.topics is a dictionary
self.api.add_topic(topic)
self.assertEquals(len(self.api.impl.topics), 1)
def test_topic_update(self):
self.test_topic_add() # updating a topic works the same like creating it
def set_up_config(self, server):
# Which components are enabled
self.component_enabled.stats = asbool(self.fs_server_config.component_enabled.stats)
self.component_enabled.slow_response = asbool(self.fs_server_config.component_enabled.slow_response)
self.component_enabled.live_msg_browser = asbool(self.fs_server_config.component_enabled.live_msg_browser)
# Details of what is enabled in live message browser
self.live_msg_browser = self.fs_server_config.live_msg_browser
self.live_msg_browser.include_internal = asbool(self.live_msg_browser.include_internal)
# Pub/sub
self.pubsub = PubSubAPI(RedisPubSub(self.kvdb.conn))
#
# Cassandra - start
#
query = self.odb.get_cassandra_conn_list(server.cluster.id, True)
self.config.cassandra_conn = ConfigDict.from_query('cassandra_conn', query)
query = self.odb.get_cassandra_query_list(server.cluster.id, True)
self.config.cassandra_query = ConfigDict.from_query('cassandra_query', query)
#
# Cassandra - end
#
#
# Search - start
#
query = self.odb.get_search_es_list(server.cluster.id, True)
self.config.search_es = ConfigDict.from_query('search_es', query)
query = self.odb.get_search_solr_list(server.cluster.id, True)
self.config.search_solr = ConfigDict.from_query('search_solr', query)
#
# Search - end
#
#
# Cloud - start
#
# OpenStack - Swift
query = self.odb.get_cloud_openstack_swift_list(server.cluster.id, True)
self.config.cloud_openstack_swift = ConfigDict.from_query('cloud_openstack_swift', query)
query = self.odb.get_cloud_aws_s3_list(server.cluster.id, True)
self.config.cloud_aws_s3 = ConfigDict.from_query('cloud_aws_s3', query)
#
# Cloud - end
#
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# Services
query = self.odb.get_service_list(server.cluster.id, True)
self.config.service = ConfigDict.from_query('service_list', query)
#
# Definitions - start
#
# AMQP
query = self.odb.get_definition_amqp_list(server.cluster.id, True)
self.config.definition_amqp = ConfigDict.from_query('definition_amqp', query)
#
# Definitions - end
#
#
# Channels - start
#
# AMQP
query = self.odb.get_channel_amqp_list(server.cluster.id, True)
self.config.channel_amqp = ConfigDict.from_query('channel_amqp', query)
# STOMP
query = self.odb.get_channel_stomp_list(server.cluster.id, True)
self.config.channel_stomp = ConfigDict.from_query('channel_stomp', query)
#
# Channels - end
#
#
# Outgoing connections - start
#
# AMQP
query = self.odb.get_out_amqp_list(server.cluster.id, True)
self.config.out_amqp = ConfigDict.from_query('out_amqp', query)
# FTP
query = self.odb.get_out_ftp_list(server.cluster.id, True)
self.config.out_ftp = ConfigDict.from_query('out_ftp', query)
# JMS WMQ
query = self.odb.get_out_jms_wmq_list(server.cluster.id, True)
self.config.out_jms_wmq = ConfigDict.from_query('out_jms_wmq', query)
# Odoo
query = self.odb.get_out_odoo_list(server.cluster.id, True)
self.config.out_odoo = ConfigDict.from_query('out_odoo', query)
# Plain HTTP
query = self.odb.get_http_soap_list(server.cluster.id, 'outgoing', 'plain_http', True)
self.config.out_plain_http = ConfigDict.from_query('out_plain_http', query)
# SOAP
query = self.odb.get_http_soap_list(server.cluster.id, 'outgoing', 'soap', True)
self.config.out_soap = ConfigDict.from_query('out_soap', query)
# SQL
query = self.odb.get_out_sql_list(server.cluster.id, True)
self.config.out_sql = ConfigDict.from_query('out_sql', query)
# STOMP
query = self.odb.get_out_stomp_list(server.cluster.id, True)
self.config.out_stomp = ConfigDict.from_query('out_stomp', query)
# ZMQ channels
query = self.odb.get_channel_zmq_list(server.cluster.id, True)
self.config.channel_zmq = ConfigDict.from_query('channel_zmq', query)
# ZMQ outgoing
query = self.odb.get_out_zmq_list(server.cluster.id, True)
self.config.out_zmq = ConfigDict.from_query('out_zmq', query)
# WebSocket channels
query = self.odb.get_channel_web_socket_list(server.cluster.id, True)
self.config.channel_web_socket = ConfigDict.from_query('channel_web_socket', query)
#
# Outgoing connections - end
#
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# Notifications - start
#
# OpenStack Swift
query = self.odb.get_notif_cloud_openstack_swift_list(server.cluster.id, True)
self.config.notif_cloud_openstack_swift = ConfigDict.from_query('notif_cloud_openstack_swift', query)
# SQL
query = self.odb.get_notif_sql_list(server.cluster.id, True)
self.config.notif_sql = ConfigDict.from_query('notif_sql', query)
#
# Notifications - end
#
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# Security - start
#
# API keys
query = self.odb.get_apikey_security_list(server.cluster.id, True)
self.config.apikey = ConfigDict.from_query('apikey', query)
# AWS
query = self.odb.get_aws_security_list(server.cluster.id, True)
self.config.aws = ConfigDict.from_query('aws', query)
# HTTP Basic Auth
query = self.odb.get_basic_auth_list(server.cluster.id, None, True)
self.config.basic_auth = ConfigDict.from_query('basic_auth', query)
# HTTP Basic Auth
query = self.odb.get_jwt_list(server.cluster.id, None, True)
self.config.jwt = ConfigDict.from_query('jwt', query)
# NTLM
query = self.odb.get_ntlm_list(server.cluster.id, True)
self.config.ntlm = ConfigDict.from_query('ntlm', query)
# OAuth
query = self.odb.get_oauth_list(server.cluster.id, True)
self.config.oauth = ConfigDict.from_query('oauth', query)
# OpenStack
query = self.odb.get_openstack_security_list(server.cluster.id, True)
self.config.openstack_security = ConfigDict.from_query('openstack_security', query)
# RBAC - permissions
query = self.odb.get_rbac_permission_list(server.cluster.id, True)
self.config.rbac_permission = ConfigDict.from_query('rbac_permission', query)
# RBAC - roles
query = self.odb.get_rbac_role_list(server.cluster.id, True)
self.config.rbac_role = ConfigDict.from_query('rbac_role', query)
# RBAC - client roles
query = self.odb.get_rbac_client_role_list(server.cluster.id, True)
self.config.rbac_client_role = ConfigDict.from_query('rbac_client_role', query)
# RBAC - role permission
query = self.odb.get_rbac_role_permission_list(server.cluster.id, True)
self.config.rbac_role_permission = ConfigDict.from_query('rbac_role_permission', query)
# Technical accounts
query = self.odb.get_tech_acc_list(server.cluster.id, True)
self.config.tech_acc = ConfigDict.from_query('tech_acc', query)
# TLS CA certs
query = self.odb.get_tls_ca_cert_list(server.cluster.id, True)
self.config.tls_ca_cert = ConfigDict.from_query('tls_ca_cert', query)
# TLS channel security
query = self.odb.get_tls_channel_sec_list(server.cluster.id, True)
self.config.tls_channel_sec = ConfigDict.from_query('tls_channel_sec', query)
# TLS key/cert pairs
query = self.odb.get_tls_key_cert_list(server.cluster.id, True)
self.config.tls_key_cert = ConfigDict.from_query('tls_key_cert', query)
# WS-Security
query = self.odb.get_wss_list(server.cluster.id, True)
self.config.wss = ConfigDict.from_query('wss', query)
# Vault connections
query = self.odb.get_vault_connection_list(server.cluster.id, True)
self.config.vault_conn_sec = ConfigDict.from_query('vault_conn_sec', query)
# XPath
query = self.odb.get_xpath_sec_list(server.cluster.id, True)
self.config.xpath_sec = ConfigDict.from_query('xpath_sec', query)
#
# Security - end
#
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# All the HTTP/SOAP channels.
http_soap = []
for item in self.odb.get_http_soap_list(server.cluster.id, 'channel'):
hs_item = {}
for key in item.keys():
hs_item[key] = getattr(item, key)
hs_item['replace_patterns_json_pointer'] = item.replace_patterns_json_pointer
hs_item['replace_patterns_xpath'] = item.replace_patterns_xpath
hs_item['match_target'] = '{}{}{}'.format(hs_item['soap_action'], MISC.SEPARATOR, hs_item['url_path'])
hs_item['match_target_compiled'] = Matcher(hs_item['match_target'])
http_soap.append(hs_item)
self.config.http_soap = http_soap
# Namespaces
query = self.odb.get_namespace_list(server.cluster.id, True)
self.config.msg_ns = ConfigDict.from_query('msg_ns', query)
# XPath
query = self.odb.get_xpath_list(server.cluster.id, True)
self.config.xpath = ConfigDict.from_query('msg_xpath', query)
# JSON Pointer
query = self.odb.get_json_pointer_list(server.cluster.id, True)
self.config.json_pointer = ConfigDict.from_query('json_pointer', query)
# SimpleIO
self.config.simple_io = ConfigDict('simple_io', Bunch())
self.config.simple_io['int_parameters'] = self.int_parameters
self.config.simple_io['int_parameter_suffixes'] = self.int_parameter_suffixes
self.config.simple_io['bool_parameter_prefixes'] = self.bool_parameter_prefixes
# Pub/sub config
self.config.pubsub = Bunch()
self.config.pubsub.default_consumer = Bunch()
self.config.pubsub.default_producer = Bunch()
query = self.odb.get_pubsub_topic_list(server.cluster.id, True)
self.config.pubsub.topics = ConfigDict.from_query('pubsub_topics', query)
id, name = self.odb.get_pubsub_default_client(server.cluster.id, 'zato.pubsub.default-consumer')
self.config.pubsub.default_consumer.id, self.config.pubsub.default_consumer.name = id, name
id, name = self.odb.get_pubsub_default_client(server.cluster.id, 'zato.pubsub.default-producer')
self.config.pubsub.default_producer.id, self.config.pubsub.default_producer.name = id, name
query = self.odb.get_pubsub_producer_list(server.cluster.id, True)
self.config.pubsub.producers = ConfigDict.from_query('pubsub_producers', query, list_config=True)
query = self.odb.get_pubsub_consumer_list(server.cluster.id, True)
self.config.pubsub.consumers = ConfigDict.from_query('pubsub_consumers', query, list_config=True)
# E-mail - SMTP
query = self.odb.get_email_smtp_list(server.cluster.id, True)
self.config.email_smtp = ConfigDict.from_query('email_smtp', query)
# E-mail - IMAP
query = self.odb.get_email_imap_list(server.cluster.id, True)
self.config.email_imap = ConfigDict.from_query('email_imap', query)
# Message paths
self.config.msg_ns_store = NamespaceStore()
self.config.json_pointer_store = JSONPointerStore()
self.config.xpath_store = XPathStore()
# Assign config to worker
self.worker_store.worker_config = self.config
self.worker_store.pubsub = self.pubsub
def _after_init_accepted(self, server, locally_deployed):
# Which components are enabled
self.component_enabled.stats = asbool(
self.fs_server_config.component_enabled.stats)
self.component_enabled.slow_response = asbool(
self.fs_server_config.component_enabled.slow_response)
# Pub/sub
self.pubsub = PubSubAPI(RedisPubSub(self.kvdb.conn))
# Repo location so that AMQP subprocesses know where to read
# the server's configuration from.
self.config.repo_location = self.repo_location
#
# Cassandra - start
#
query = self.odb.get_cassandra_conn_list(server.cluster.id, True)
self.config.cassandra_conn = ConfigDict.from_query(
'cassandra_conn', query)
query = self.odb.get_cassandra_query_list(server.cluster.id, True)
self.config.cassandra_query = ConfigDict.from_query(
'cassandra_query', query)
#
# Cassandra - end
#
#
# Search - start
#
query = self.odb.get_search_es_list(server.cluster.id, True)
self.config.search_es = ConfigDict.from_query('search_es', query)
query = self.odb.get_search_solr_list(server.cluster.id, True)
self.config.search_solr = ConfigDict.from_query('search_solr', query)
#
# Search - end
#
#
# Cloud - start
#
# OpenStack - Swift
query = self.odb.get_cloud_openstack_swift_list(
server.cluster.id, True)
self.config.cloud_openstack_swift = ConfigDict.from_query(
'cloud_openstack_swift', query)
query = self.odb.get_cloud_aws_s3_list(server.cluster.id, True)
self.config.cloud_aws_s3 = ConfigDict.from_query('cloud_aws_s3', query)
#
# Cloud - end
#
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# Services
query = self.odb.get_service_list(server.cluster.id, True)
self.config.service = ConfigDict.from_query('service_list', query)
#
# Channels - start
#
# STOMP
query = self.odb.get_channel_stomp_list(server.cluster.id, True)
self.config.channel_stomp = ConfigDict.from_query(
'channel_stomp', query)
#
# Channels - end
#
#
# Outgoing connections - start
#
# AMQP
query = self.odb.get_out_amqp_list(server.cluster.id, True)
self.config.out_amqp = ConfigDict.from_query('out_amqp', query)
# FTP
query = self.odb.get_out_ftp_list(server.cluster.id, True)
self.config.out_ftp = ConfigDict.from_query('out_ftp', query)
# JMS WMQ
query = self.odb.get_out_jms_wmq_list(server.cluster.id, True)
self.config.out_jms_wmq = ConfigDict.from_query('out_jms_wmq', query)
# Odoo
query = self.odb.get_out_odoo_list(server.cluster.id, True)
self.config.out_odoo = ConfigDict.from_query('out_odoo', query)
# Plain HTTP
query = self.odb.get_http_soap_list(server.cluster.id, 'outgoing',
'plain_http', True)
self.config.out_plain_http = ConfigDict.from_query(
'out_plain_http', query)
# SOAP
query = self.odb.get_http_soap_list(server.cluster.id, 'outgoing',
'soap', True)
self.config.out_soap = ConfigDict.from_query('out_soap', query)
# SQL
query = self.odb.get_out_sql_list(server.cluster.id, True)
self.config.out_sql = ConfigDict.from_query('out_sql', query)
# STOMP
query = self.odb.get_out_stomp_list(server.cluster.id, True)
self.config.out_stomp = ConfigDict.from_query('out_stomp', query)
# ZMQ channels
query = self.odb.get_channel_zmq_list(server.cluster.id, True)
self.config.channel_zmq = ConfigDict.from_query('channel_zmq', query)
# ZMQ outgoing
query = self.odb.get_out_zmq_list(server.cluster.id, True)
self.config.out_zmq = ConfigDict.from_query('out_zmq', query)
# WebSocket channels
query = self.odb.get_channel_web_socket_list(server.cluster.id, True)
self.config.channel_web_socket = ConfigDict.from_query(
'channel_web_socket', query)
#
# Outgoing connections - end
#
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# Notifications - start
#
# OpenStack Swift
query = self.odb.get_notif_cloud_openstack_swift_list(
server.cluster.id, True)
self.config.notif_cloud_openstack_swift = ConfigDict.from_query(
'notif_cloud_openstack_swift', query)
# SQL
query = self.odb.get_notif_sql_list(server.cluster.id, True)
self.config.notif_sql = ConfigDict.from_query('notif_sql', query)
#
# Notifications - end
#
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# Security - start
#
# API keys
query = self.odb.get_apikey_security_list(server.cluster.id, True)
self.config.apikey = ConfigDict.from_query('apikey', query)
# AWS
query = self.odb.get_aws_security_list(server.cluster.id, True)
self.config.aws = ConfigDict.from_query('aws', query)
# HTTP Basic Auth
query = self.odb.get_basic_auth_list(server.cluster.id, None, True)
self.config.basic_auth = ConfigDict.from_query('basic_auth', query)
# HTTP Basic Auth
query = self.odb.get_jwt_list(server.cluster.id, None, True)
self.config.jwt = ConfigDict.from_query('jwt', query)
# NTLM
query = self.odb.get_ntlm_list(server.cluster.id, True)
self.config.ntlm = ConfigDict.from_query('ntlm', query)
# OAuth
query = self.odb.get_oauth_list(server.cluster.id, True)
self.config.oauth = ConfigDict.from_query('oauth', query)
# OpenStack
query = self.odb.get_openstack_security_list(server.cluster.id, True)
self.config.openstack_security = ConfigDict.from_query(
'openstack_security', query)
# RBAC - permissions
query = self.odb.get_rbac_permission_list(server.cluster.id, True)
self.config.rbac_permission = ConfigDict.from_query(
'rbac_permission', query)
# RBAC - roles
query = self.odb.get_rbac_role_list(server.cluster.id, True)
self.config.rbac_role = ConfigDict.from_query('rbac_role', query)
# RBAC - client roles
query = self.odb.get_rbac_client_role_list(server.cluster.id, True)
self.config.rbac_client_role = ConfigDict.from_query(
'rbac_client_role', query)
# RBAC - role permission
query = self.odb.get_rbac_role_permission_list(server.cluster.id, True)
self.config.rbac_role_permission = ConfigDict.from_query(
'rbac_role_permission', query)
# Technical accounts
query = self.odb.get_tech_acc_list(server.cluster.id, True)
self.config.tech_acc = ConfigDict.from_query('tech_acc', query)
# TLS CA certs
query = self.odb.get_tls_ca_cert_list(server.cluster.id, True)
self.config.tls_ca_cert = ConfigDict.from_query('tls_ca_cert', query)
# TLS channel security
query = self.odb.get_tls_channel_sec_list(server.cluster.id, True)
self.config.tls_channel_sec = ConfigDict.from_query(
'tls_channel_sec', query)
# TLS key/cert pairs
query = self.odb.get_tls_key_cert_list(server.cluster.id, True)
self.config.tls_key_cert = ConfigDict.from_query('tls_key_cert', query)
# WS-Security
query = self.odb.get_wss_list(server.cluster.id, True)
self.config.wss = ConfigDict.from_query('wss', query)
# XPath
query = self.odb.get_xpath_sec_list(server.cluster.id, True)
self.config.xpath_sec = ConfigDict.from_query('xpath_sec', query)
#
# Security - end
#
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# All the HTTP/SOAP channels.
http_soap = []
for item in self.odb.get_http_soap_list(server.cluster.id, 'channel'):
hs_item = Bunch()
for key in item.keys():
hs_item[key] = getattr(item, key)
hs_item.replace_patterns_json_pointer = item.replace_patterns_json_pointer
hs_item.replace_patterns_xpath = item.replace_patterns_xpath
hs_item.match_target = '{}{}{}'.format(hs_item.soap_action,
MISC.SEPARATOR,
hs_item.url_path)
hs_item.match_target_compiled = Matcher(hs_item.match_target)
http_soap.append(hs_item)
self.config.http_soap = http_soap
# Namespaces
query = self.odb.get_namespace_list(server.cluster.id, True)
self.config.msg_ns = ConfigDict.from_query('msg_ns', query)
# XPath
query = self.odb.get_xpath_list(server.cluster.id, True)
self.config.xpath = ConfigDict.from_query('msg_xpath', query)
# JSON Pointer
query = self.odb.get_json_pointer_list(server.cluster.id, True)
self.config.json_pointer = ConfigDict.from_query('json_pointer', query)
# SimpleIO
self.config.simple_io = ConfigDict('simple_io', Bunch())
self.config.simple_io['int_parameters'] = self.int_parameters
self.config.simple_io[
'int_parameter_suffixes'] = self.int_parameter_suffixes
self.config.simple_io[
'bool_parameter_prefixes'] = self.bool_parameter_prefixes
# Pub/sub config
self.config.pubsub = Bunch()
self.config.pubsub.default_consumer = Bunch()
self.config.pubsub.default_producer = Bunch()
query = self.odb.get_pubsub_topic_list(server.cluster.id, True)
self.config.pubsub.topics = ConfigDict.from_query(
'pubsub_topics', query)
id, name = self.odb.get_pubsub_default_client(
server.cluster.id, 'zato.pubsub.default-consumer')
self.config.pubsub.default_consumer.id, self.config.pubsub.default_consumer.name = id, name
id, name = self.odb.get_pubsub_default_client(
server.cluster.id, 'zato.pubsub.default-producer')
self.config.pubsub.default_producer.id, self.config.pubsub.default_producer.name = id, name
query = self.odb.get_pubsub_producer_list(server.cluster.id, True)
self.config.pubsub.producers = ConfigDict.from_query(
'pubsub_producers', query, list_config=True)
query = self.odb.get_pubsub_consumer_list(server.cluster.id, True)
self.config.pubsub.consumers = ConfigDict.from_query(
'pubsub_consumers', query, list_config=True)
# E-mail - SMTP
query = self.odb.get_email_smtp_list(server.cluster.id, True)
self.config.email_smtp = ConfigDict.from_query('email_smtp', query)
# E-mail - IMAP
query = self.odb.get_email_imap_list(server.cluster.id, True)
self.config.email_imap = ConfigDict.from_query('email_imap', query)
# Assign config to worker
self.worker_store.worker_config = self.config
self.worker_store.pubsub = self.pubsub
self.worker_store.init()
# Deployed missing services found on other servers
if locally_deployed:
self.deploy_missing_services(locally_deployed)
# Signal to ODB that we are done with deploying everything
self.odb.on_deployment_finished()
# Default content type
self.json_content_type = self.fs_server_config.content_type.json
self.plain_xml_content_type = self.fs_server_config.content_type.plain_xml
self.soap11_content_type = self.fs_server_config.content_type.soap11
self.soap12_content_type = self.fs_server_config.content_type.soap12
