class RedisPubSubTestCase(RedisPubSubCommonTestCase): def setUp(self): super(RedisPubSubTestCase, self).setUp() self.api = PubSubAPI(RedisPubSub(self.kvdb, self.key_prefix)) # ################################################################################################################################ def _publish_move(self, move=True, **kwargs): payload = rand_string() topic = Topic(rand_string()) self.api.add_topic(topic) producer = Client(rand_int(), rand_string()) self.api.add_producer(producer, topic) ctx = self.api.publish(payload, topic.name, client_id=producer.id, **kwargs) if move: self.api.impl.move_to_target_queues() return payload, topic, producer, ctx def _check_publish(self, **kwargs): if kwargs: expected_mime_type = kwargs['mime_type'] expected_priority = kwargs['priority'] expected_expiration = kwargs['expiration'] else: expected_mime_type = PUB_SUB.DEFAULT_MIME_TYPE expected_priority = PUB_SUB.DEFAULT_PRIORITY expected_expiration = PUB_SUB.DEFAULT_EXPIRATION payload, topic, producer, ctx = self._publish_move(**kwargs) now = datetime.utcnow() # ######################################################################################################################## # # MSG_METADATA_KEY # # ######################################################################################################################## msg_metadata_dict = self.kvdb.hgetall(self.api.impl.MSG_METADATA_KEY) # E.g. {'K0321C8Q5X67N7K2D642ZYZCXY5T': '{"topic": "ab3ee73838d174cd690a1947b56f67674", "priority": 5, "expiration": 60.0, # "producer": "a951ec619d0f449969529c0bfe8f7900f", # "creation_time_utc": "2014-04-06T19:51:37.784905", "msg_id": "K0321C8Q5X67N7K2D642ZYZCXY5T", # "expire_at_utc": "2014-04-06T19:52:37.784905", "mime_type": "text/plain"}'} self.assertEquals(len(msg_metadata_dict), 1) self.assertTrue(ctx.msg.msg_id in msg_metadata_dict) msg_metadata = loads(msg_metadata_dict[ctx.msg.msg_id]) self.assertEquals(msg_metadata['mime_type'], expected_mime_type) self.assertEquals(msg_metadata['priority'], expected_priority) self.assertEquals(msg_metadata['expiration'], expected_expiration) self.assertEquals(msg_metadata['topic'], topic.name) self.assertEquals(msg_metadata['producer'], producer.name) creation_time_utc = parse(msg_metadata['creation_time_utc']) expire_at_utc = parse(msg_metadata['expire_at_utc']) self.assertTrue( creation_time_utc < now, 'creation_time_utc:`{}` is not less than now:`{}`'.format( creation_time_utc, now)) self.assertTrue( expire_at_utc > now, 'creation_time_utc:`{}` is not greater than now:`{}`'.format( expire_at_utc, now)) # ######################################################################################################################## # # LAST_PUB_TIME_KEY # # ######################################################################################################################## last_pub_time = self.kvdb.hgetall(self.api.impl.LAST_PUB_TIME_KEY) self.assertEquals(len(last_pub_time), 1) last_pub_time = parse(last_pub_time[topic.name]) self.assertTrue( last_pub_time < now, 'last_pub_time:`{}` is not less than now:`{}`'.format( last_pub_time, now)) # ######################################################################################################################## # # MSG_EXPIRE_AT_KEY # # ######################################################################################################################## msg_expire_at = self.kvdb.hgetall(self.api.impl.MSG_EXPIRE_AT_KEY) self.assertEquals(len(msg_expire_at), 1) msg_expire_at = parse(msg_expire_at[ctx.msg.msg_id]) self.assertTrue( msg_expire_at > now, 'msg_expire_at:`{}` is not greater than now:`{}`'.format( msg_expire_at, now)) # ######################################################################################################################## # # LAST_SEEN_PRODUCER_KEY # # ######################################################################################################################## last_seen_producer = self.kvdb.hgetall( self.api.impl.LAST_SEEN_PRODUCER_KEY) self.assertEquals(len(last_seen_producer), 1) last_seen_producer = parse(last_seen_producer[str(producer.id)]) self.assertTrue( last_seen_producer < now, 'last_seen_producer:`{}` is not less than now:`{}`'.format( last_seen_producer, now)) # ######################################################################################################################## # # MSG_VALUES_KEY # # ######################################################################################################################## msg_values = self.kvdb.hgetall(self.api.impl.MSG_VALUES_KEY) self.assertEquals(len(msg_values), 1) self.assertEquals(payload, msg_values[ctx.msg.msg_id]) def test_publish_defaults(self): self._check_publish() def test_publish_custom_attrs(self): self._check_publish( **{ 'mime_type': rand_string(), 'priority': rand_int(), 'expiration': rand_int(1000, 2000), 'msg_id': rand_string(), }) # ################################################################################################################################ def test_delete_metadata(self): payload, topic, producer, ctx = self._publish_move(move=False) consumer = Consumer(rand_int(), rand_string()) self.api.add_consumer(consumer, topic) sub_key = self.api.subscribe(consumer.id, topic.name) self.api.impl.move_to_target_queues() self._check_consumer_queue_before_get(ctx, sub_key) self._check_get(ctx, sub_key, topic, producer, consumer) self.api.acknowledge(sub_key, ctx.msg.msg_id) # Ok, we should now have metadata for the consumer, producer and topic. last_seen_consumer = self.api.impl.kvdb.hkeys( self.api.impl.LAST_SEEN_CONSUMER_KEY) last_seen_producer = self.api.impl.kvdb.hkeys( self.api.impl.LAST_SEEN_PRODUCER_KEY) last_pub_time = self.api.impl.kvdb.hkeys( self.api.impl.LAST_PUB_TIME_KEY) self.assertIn(str(consumer.id), last_seen_consumer) self.assertIn(str(producer.id), last_seen_producer) self.assertIn(topic.name, last_pub_time) self.api.impl.delete_producer(producer, topic) last_seen_producer = self.api.impl.kvdb.hkeys( self.api.impl.LAST_SEEN_PRODUCER_KEY) self.assertNotIn(str(producer.id), last_seen_producer) self.api.impl.delete_consumer(consumer, topic) last_seen_consumer = self.api.impl.kvdb.hkeys( self.api.impl.LAST_SEEN_CONSUMER_KEY) self.assertNotIn(str(consumer.id), last_seen_consumer) self.api.impl.delete_topic(topic) last_pub_time = self.api.impl.kvdb.hkeys( self.api.impl.LAST_PUB_TIME_KEY) self.assertNotIn(topic.name, last_pub_time) # ################################################################################################################################ def test_subscribe(self): client_id, client_name = rand_int(), rand_string() client = Client(client_id, client_name) topics = rand_string(rand_int()) sub_key = self.api.subscribe(client.id, topics) self.assertEquals(self.api.impl.sub_to_cons[sub_key], client_id) self.assertEquals(self.api.impl.cons_to_sub[client_id], sub_key) self.assertEquals(sorted(self.api.impl.cons_to_topic[client_id]), sorted(topics)) for topic in topics: self.assertIn(client_id, self.api.impl.topic_to_cons[topic]) # ################################################################################################################################ def _check_consumer_queue_before_get(self, ctx, sub_key): # ######################################################################################################################## # # UNACK_COUNTER_KEY # # ######################################################################################################################## unack_counter = self.kvdb.hgetall(self.api.impl.UNACK_COUNTER_KEY) self.assertEquals(len(unack_counter), 1) self.assertEqual(unack_counter[ctx.msg.msg_id], '1') # One subscriber hence one undelivered message # ######################################################################################################################## # # CONSUMER_MSG_IDS_PREFIX # # ######################################################################################################################## consumer_msg_ids = self.kvdb.lrange( self.api.impl.CONSUMER_MSG_IDS_PREFIX.format(sub_key), 0, -1) self.assertEquals(consumer_msg_ids, [ctx.msg.msg_id]) def _check_get(self, ctx, sub_key, topic, producer, client): msg = list(self.api.get(sub_key))[0].to_dict() self.assertEquals(msg['topic'], topic.name) self.assertEquals(msg['priority'], PUB_SUB.DEFAULT_PRIORITY) self.assertEquals(msg['expiration'], PUB_SUB.DEFAULT_EXPIRATION) self.assertEquals(msg['producer'], producer.name) self.assertEquals(msg['msg_id'], ctx.msg.msg_id) self.assertEquals(msg['mime_type'], PUB_SUB.DEFAULT_MIME_TYPE) now = datetime.utcnow() creation_time_utc = parse(msg['creation_time_utc']) expire_at_utc = parse(msg['expire_at_utc']) self.assertTrue( creation_time_utc < now, 'creation_time_utc:`{}` is not less than now:`{}`'.format( creation_time_utc, now)) self.assertTrue( expire_at_utc > now, 'creation_time_utc:`{}` is not greater than now:`{}`'.format( expire_at_utc, now)) # ######################################################################################################################## # # LAST_SEEN_CONSUMER_KEY # # ######################################################################################################################## last_seen_consumer = self.kvdb.hgetall( self.api.impl.LAST_SEEN_CONSUMER_KEY) self.assertEquals(len(last_seen_consumer), 1) last_seen_consumer = parse(last_seen_consumer[str(client.id)]) self.assertTrue( last_seen_consumer < now, 'last_seen_consumer:`{}` is not less than now:`{}`'.format( last_seen_consumer, now)) # ######################################################################################################################## # # CONSUMER_IN_FLIGHT_IDS_PREFIX # # ######################################################################################################################## consumer_id_flight_ids = self.kvdb.smembers( self.api.impl.CONSUMER_IN_FLIGHT_IDS_PREFIX.format(sub_key)) self.assertEquals(len(consumer_id_flight_ids), 1) self.assertEqual(list(consumer_id_flight_ids), [ctx.msg.msg_id]) # ######################################################################################################################## # # CONSUMER_IN_FLIGHT_DATA_PREFIX # # ######################################################################################################################## consumer_in_flight_data = self.kvdb.hgetall( self.api.impl.CONSUMER_IN_FLIGHT_DATA_PREFIX.format(sub_key)) self.assertEquals(len(consumer_in_flight_data), 1) consumer_in_flight_data = parse( consumer_in_flight_data[ctx.msg.msg_id]) self.assertTrue( consumer_in_flight_data < now, 'consumer_in_flight_data:`{}` is not less than now:`{}`'.format( consumer_in_flight_data, now)) # There should still be one unacknowledged message. unack_counter = self.kvdb.hgetall(self.api.impl.UNACK_COUNTER_KEY) self.assertEquals(len(unack_counter), 1) self.assertEqual(unack_counter[ctx.msg.msg_id], '1') # One subscriber hence one undelivered message def test_get_reject_acknowledge(self): payload, topic, producer, ctx = self._publish_move(move=False) client_id, client_name = rand_int(), rand_string() client = Client(client_id, client_name) sub_key = self.api.subscribe(client.id, topic.name) # Moves a message to the consumer's queue self.api.impl.move_to_target_queues() self._check_consumer_queue_before_get(ctx, sub_key) # Consumer gets a message which puts it in the in-flight state. self._check_get(ctx, sub_key, topic, producer, client) # However, there should be nothing in the consumer's queue. consumer_msg_ids = self.kvdb.lrange( self.api.impl.CONSUMER_MSG_IDS_PREFIX.format(sub_key), 0, -1) self.assertEquals(consumer_msg_ids, []) # Consumer rejects the message which puts it back on a queue. self.api.reject(sub_key, ctx.msg.msg_id) # After rejection it's as though the message has just been published. self._check_consumer_queue_before_get(ctx, sub_key) # Get after rejection works as before. self._check_get(ctx, sub_key, topic, producer, client) # Consumer acknowledges a message. self.api.acknowledge(sub_key, ctx.msg.msg_id) # This was the only one subscription so now that the message has been delivered # there should be no trace of it in backend. # The only keys left are LAST_PUB_TIME_KEY, LAST_SEEN_CONSUMER_KEY and LAST_SEEN_PRODUCER_KEY - nothing else. keys = self.kvdb.keys('{}*'.format(self.key_prefix)) self.assertEquals(len(keys), 3) now = datetime.utcnow() last_pub_time = parse( self.kvdb.hgetall(self.api.impl.LAST_PUB_TIME_KEY)[topic.name]) last_seen_consumer = parse( self.kvdb.hgetall(self.api.impl.LAST_SEEN_CONSUMER_KEY)[str( client.id)]) last_seen_producer = parse( self.kvdb.hgetall(self.api.impl.LAST_SEEN_PRODUCER_KEY)[str( producer.id)]) self.assertTrue( last_pub_time < now, 'last_pub_time:`{}` is not less than now:`{}`'.format( last_pub_time, now)) self.assertTrue( last_seen_consumer < now, 'last_seen_consumer:`{}` is not less than now:`{}`'.format( last_seen_consumer, now)) self.assertTrue( last_seen_producer < now, 'last_seen_producer:`{}` is not less than now:`{}`'.format( last_seen_producer, now)) # ################################################################################################################################ def test_pub_sub_exception(self): invalid_sub_key = rand_string() valid_sub_key = rand_string() client_id = rand_int() topic_name = rand_string() consumer = Consumer(client_id, rand_string(), sub_key=valid_sub_key) topic = Topic(topic_name) # Without adding consumer key, validation won't succeed. self.assertRaises(PubSubException, self.api.impl.validate_sub_key, invalid_sub_key) self.assertRaises(PubSubException, self.api.impl.validate_sub_key, valid_sub_key) # After adding a subscription key no error should be raised. self.api.impl.add_consumer(consumer, topic) self.api.impl.add_subscription(valid_sub_key, client_id, topic_name) self.assertRaises(PubSubException, self.api.impl.validate_sub_key, invalid_sub_key) self.api.impl.validate_sub_key( valid_sub_key) # Should not raise any exception now. self.api.impl.delete_consumer(consumer, topic) # After deleting a consumer, validation won't succeed anymore. self.assertRaises(PubSubException, self.api.impl.validate_sub_key, invalid_sub_key) self.assertRaises(PubSubException, self.api.impl.validate_sub_key, valid_sub_key) def invoke_func_sub_key(func, sub_key, *args): list(func(sub_key, *args)) self.assertRaises(PubSubException, invoke_func_sub_key, self.api.get, valid_sub_key) self.assertRaises(PubSubException, invoke_func_sub_key, self.api.get, invalid_sub_key) self.assertRaises(PubSubException, invoke_func_sub_key, self.api.acknowledge, valid_sub_key, 'abc') self.assertRaises(PubSubException, invoke_func_sub_key, self.api.acknowledge, invalid_sub_key, 'def') self.assertRaises(PubSubException, invoke_func_sub_key, self.api.reject, valid_sub_key, 'abc') self.assertRaises(PubSubException, invoke_func_sub_key, self.api.reject, invalid_sub_key, 'def') def test_publish_exceptions(self): payload = rand_string() producer = Client(rand_int(), rand_string()) def invoke_publish(payload, topic, producer_id): self.api.publish(payload, topic, client_id=producer_id) # KeyError because no such producer is in self.api.impl.producers. self.assertRaises(KeyError, invoke_publish, payload, rand_string(), producer.id) # Adding a producer but still, no such topic. self.api.add_producer(producer, Topic(rand_string())) self.assertRaises(PubSubException, invoke_publish, payload, rand_string(), producer.id) # Adding a topic but still PubSubException is raised because the producer is not allowed to use it. topic = Topic(rand_string()) self.api.add_topic(topic) self.assertRaises(PubSubException, invoke_publish, payload, topic.name, producer.id) # Combining the topic and producer, no exception is raised now. self.api.add_producer(producer, topic) invoke_publish(payload, topic.name, producer.id) # But it's not possible to publish to inactive topics. self.api.impl.topics[topic.name].is_active = False self.assertRaises(PubSubException, invoke_publish, payload, topic.name, producer.id) # Make the topic active and it can be published to again. self.api.impl.topics[topic.name].is_active = True invoke_publish(payload, topic.name, producer.id) # Inactive producers cannot publish to topics either. self.api.impl.producers[producer.id].is_active = False self.assertRaises(PubSubException, invoke_publish, payload, topic.name, producer.id) # Making them active means they can publish again. self.api.impl.producers[producer.id].is_active = True invoke_publish(payload, topic.name, producer.id) def test_ping(self): response = self.api.impl.ping() self.assertIsInstance(response, bool) self.assertEquals(response, True) # ################################################################################################################################ def test_default_clients(self): # Initially, default clients are dummy ones. default_consumer = self.api.get_default_consumer() default_producer = self.api.get_default_producer() self.assertEquals(default_consumer.id, None) self.assertEquals(default_consumer.name, None) self.assertEquals(default_consumer.is_active, True) self.assertEquals(default_producer.id, None) self.assertEquals(default_producer.name, None) self.assertEquals(default_producer.is_active, True) cons_id = rand_int() cons_name = rand_string() cons_is_active = rand_bool() prod_name = rand_string() prod_id = rand_int() prod_is_active = rand_bool() cons = Client(cons_id, cons_name, cons_is_active) prod = Client(prod_id, prod_name, prod_is_active) self.api.set_default_consumer(cons) self.api.set_default_producer(prod) default_consumer = self.api.get_default_consumer() default_producer = self.api.get_default_producer() self.assertEquals(default_consumer.id, cons_id) self.assertEquals(default_consumer.name, cons_name) self.assertEquals(default_consumer.is_active, cons_is_active) self.assertEquals(default_producer.id, prod_id) self.assertEquals(default_producer.name, prod_name) self.assertEquals(default_producer.is_active, prod_is_active) # ################################################################################################################################ def test_topic_add(self): name = rand_string() is_active = rand_bool() is_fifo = rand_bool() max_depth = rand_int() topic = Topic(name, is_active, is_fifo, max_depth) self.api.add_topic(topic) self.assertIn(name, self.api.impl.topics) self.assertEquals(len(self.api.impl.topics), 1) given = self.api.impl.topics[name] self.assertEquals(given.name, name) self.assertEquals(given.is_active, is_active) self.assertEquals(given.is_fifo, is_fifo) self.assertEquals(given.max_depth, max_depth) # Adding topic of the same name should not create a new topic because impl.topics is a dictionary self.api.add_topic(topic) self.assertEquals(len(self.api.impl.topics), 1) def test_topic_update(self): self.test_topic_add( ) # updating a topic works the same like creating it
class RedisPubSubTestCase(RedisPubSubCommonTestCase): def setUp(self): super(RedisPubSubTestCase, self).setUp() self.api = PubSubAPI(RedisPubSub(self.kvdb, self.key_prefix)) # ################################################################################################################################ def _publish_move(self, move=True, **kwargs): payload = rand_string() topic = Topic(rand_string()) self.api.add_topic(topic) producer = Client(rand_int(), rand_string()) self.api.add_producer(producer, topic) ctx = self.api.publish(payload, topic.name, client_id=producer.id, **kwargs) if move: self.api.impl.move_to_target_queues() return payload, topic, producer, ctx def _check_publish(self, **kwargs): if kwargs: expected_mime_type = kwargs['mime_type'] expected_priority = kwargs['priority'] expected_expiration = kwargs['expiration'] expected_msg_id = kwargs['msg_id'] else: expected_mime_type = PUB_SUB.DEFAULT_MIME_TYPE expected_priority = PUB_SUB.DEFAULT_PRIORITY expected_expiration = PUB_SUB.DEFAULT_EXPIRATION expected_msg_id = None payload, topic, producer, ctx = self._publish_move(**kwargs) now = datetime.utcnow() # ######################################################################################################################## # # MSG_METADATA_KEY # # ######################################################################################################################## msg_metadata_dict = self.kvdb.hgetall(self.api.impl.MSG_METADATA_KEY) # E.g. {'K0321C8Q5X67N7K2D642ZYZCXY5T': '{"topic": "ab3ee73838d174cd690a1947b56f67674", "priority": 5, "expiration": 60.0, # "producer": "a951ec619d0f449969529c0bfe8f7900f", # "creation_time_utc": "2014-04-06T19:51:37.784905", "msg_id": "K0321C8Q5X67N7K2D642ZYZCXY5T", # "expire_at_utc": "2014-04-06T19:52:37.784905", "mime_type": "text/plain"}'} self.assertEquals(len(msg_metadata_dict), 1) self.assertTrue(ctx.msg.msg_id in msg_metadata_dict) msg_metadata = loads(msg_metadata_dict[ctx.msg.msg_id]) self.assertEquals(msg_metadata['mime_type'], expected_mime_type) self.assertEquals(msg_metadata['priority'], expected_priority) self.assertEquals(msg_metadata['expiration'], expected_expiration) self.assertEquals(msg_metadata['topic'], topic.name) self.assertEquals(msg_metadata['producer'], producer.name) creation_time_utc = parse(msg_metadata['creation_time_utc']) expire_at_utc = parse(msg_metadata['expire_at_utc']) self.assertTrue(creation_time_utc < now, 'creation_time_utc:`{}` is not less than now:`{}`'.format(creation_time_utc, now)) self.assertTrue(expire_at_utc > now, 'creation_time_utc:`{}` is not greater than now:`{}`'.format(expire_at_utc, now)) # ######################################################################################################################## # # LAST_PUB_TIME_KEY # # ######################################################################################################################## last_pub_time = self.kvdb.hgetall(self.api.impl.LAST_PUB_TIME_KEY) self.assertEquals(len(last_pub_time), 1) last_pub_time = parse(last_pub_time[topic.name]) self.assertTrue(last_pub_time < now, 'last_pub_time:`{}` is not less than now:`{}`'.format(last_pub_time, now)) # ######################################################################################################################## # # MSG_EXPIRE_AT_KEY # # ######################################################################################################################## msg_expire_at = self.kvdb.hgetall(self.api.impl.MSG_EXPIRE_AT_KEY) self.assertEquals(len(msg_expire_at), 1) msg_expire_at = parse(msg_expire_at[ctx.msg.msg_id]) self.assertTrue(msg_expire_at > now, 'msg_expire_at:`{}` is not greater than now:`{}`'.format(msg_expire_at, now)) # ######################################################################################################################## # # LAST_SEEN_PRODUCER_KEY # # ######################################################################################################################## last_seen_producer = self.kvdb.hgetall(self.api.impl.LAST_SEEN_PRODUCER_KEY) self.assertEquals(len(last_seen_producer), 1) last_seen_producer = parse(last_seen_producer[str(producer.id)]) self.assertTrue(last_seen_producer < now, 'last_seen_producer:`{}` is not less than now:`{}`'.format(last_seen_producer, now)) # ######################################################################################################################## # # MSG_VALUES_KEY # # ######################################################################################################################## msg_values = self.kvdb.hgetall(self.api.impl.MSG_VALUES_KEY) self.assertEquals(len(msg_values), 1) self.assertEquals(payload, msg_values[ctx.msg.msg_id]) def test_publish_defaults(self): self._check_publish() def test_publish_custom_attrs(self): self._check_publish(**{ 'mime_type': rand_string(), 'priority': rand_int(), 'expiration': rand_int(1000, 2000), 'msg_id': rand_string(), }) # ################################################################################################################################ def test_delete_metadata(self): payload, topic, producer, ctx = self._publish_move(move=False) consumer = Consumer(rand_int(), rand_string()) self.api.add_consumer(consumer, topic) sub_key = self.api.subscribe(consumer.id, topic.name) self.api.impl.move_to_target_queues() self._check_consumer_queue_before_get(ctx, sub_key) self._check_get(ctx, sub_key, topic, producer, consumer) self.api.acknowledge(sub_key, ctx.msg.msg_id) # Ok, we should now have metadata for the consumer, producer and topic. last_seen_consumer = self.api.impl.kvdb.hkeys(self.api.impl.LAST_SEEN_CONSUMER_KEY) last_seen_producer = self.api.impl.kvdb.hkeys(self.api.impl.LAST_SEEN_PRODUCER_KEY) last_pub_time = self.api.impl.kvdb.hkeys(self.api.impl.LAST_PUB_TIME_KEY) self.assertIn(str(consumer.id), last_seen_consumer) self.assertIn(str(producer.id), last_seen_producer) self.assertIn(topic.name, last_pub_time) self.api.impl.delete_producer(producer, topic) last_seen_producer = self.api.impl.kvdb.hkeys(self.api.impl.LAST_SEEN_PRODUCER_KEY) self.assertNotIn(str(producer.id), last_seen_producer) self.api.impl.delete_consumer(consumer, topic) last_seen_consumer = self.api.impl.kvdb.hkeys(self.api.impl.LAST_SEEN_CONSUMER_KEY) self.assertNotIn(str(consumer.id), last_seen_consumer) self.api.impl.delete_topic(topic) last_pub_time = self.api.impl.kvdb.hkeys(self.api.impl.LAST_PUB_TIME_KEY) self.assertNotIn(topic.name, last_pub_time) # ################################################################################################################################ def test_subscribe(self): client_id, client_name = rand_int(), rand_string() client = Client(client_id, client_name) topics = rand_string(rand_int()) sub_key = self.api.subscribe(client.id, topics) self.assertEquals(self.api.impl.sub_to_cons[sub_key], client_id) self.assertEquals(self.api.impl.cons_to_sub[client_id], sub_key) self.assertEquals(sorted(self.api.impl.cons_to_topic[client_id]), sorted(topics)) for topic in topics: self.assertIn(client_id, self.api.impl.topic_to_cons[topic]) # ################################################################################################################################ def _check_consumer_queue_before_get(self, ctx, sub_key): # ######################################################################################################################## # # UNACK_COUNTER_KEY # # ######################################################################################################################## unack_counter = self.kvdb.hgetall(self.api.impl.UNACK_COUNTER_KEY) self.assertEquals(len(unack_counter), 1) self.assertEqual(unack_counter[ctx.msg.msg_id], '1') # One subscriber hence one undelivered message # ######################################################################################################################## # # CONSUMER_MSG_IDS_PREFIX # # ######################################################################################################################## consumer_msg_ids = self.kvdb.lrange(self.api.impl.CONSUMER_MSG_IDS_PREFIX.format(sub_key), 0, -1) self.assertEquals(consumer_msg_ids, [ctx.msg.msg_id]) def _check_get(self, ctx, sub_key, topic, producer, client): msg = list(self.api.get(sub_key))[0].to_dict() self.assertEquals(msg['topic'], topic.name) self.assertEquals(msg['priority'], PUB_SUB.DEFAULT_PRIORITY) self.assertEquals(msg['expiration'], PUB_SUB.DEFAULT_EXPIRATION) self.assertEquals(msg['producer'], producer.name) self.assertEquals(msg['msg_id'], ctx.msg.msg_id) self.assertEquals(msg['mime_type'], PUB_SUB.DEFAULT_MIME_TYPE) now = datetime.utcnow() creation_time_utc = parse(msg['creation_time_utc']) expire_at_utc = parse(msg['expire_at_utc']) self.assertTrue(creation_time_utc < now, 'creation_time_utc:`{}` is not less than now:`{}`'.format(creation_time_utc, now)) self.assertTrue(expire_at_utc > now, 'creation_time_utc:`{}` is not greater than now:`{}`'.format(expire_at_utc, now)) # ######################################################################################################################## # # LAST_SEEN_CONSUMER_KEY # # ######################################################################################################################## last_seen_consumer = self.kvdb.hgetall(self.api.impl.LAST_SEEN_CONSUMER_KEY) self.assertEquals(len(last_seen_consumer), 1) last_seen_consumer = parse(last_seen_consumer[str(client.id)]) self.assertTrue(last_seen_consumer < now, 'last_seen_consumer:`{}` is not less than now:`{}`'.format(last_seen_consumer, now)) # ######################################################################################################################## # # CONSUMER_IN_FLIGHT_IDS_PREFIX # # ######################################################################################################################## consumer_id_flight_ids = self.kvdb.smembers(self.api.impl.CONSUMER_IN_FLIGHT_IDS_PREFIX.format(sub_key)) self.assertEquals(len(consumer_id_flight_ids), 1) self.assertEqual(list(consumer_id_flight_ids), [ctx.msg.msg_id]) # ######################################################################################################################## # # CONSUMER_IN_FLIGHT_DATA_PREFIX # # ######################################################################################################################## consumer_in_flight_data = self.kvdb.hgetall(self.api.impl.CONSUMER_IN_FLIGHT_DATA_PREFIX.format(sub_key)) self.assertEquals(len(consumer_in_flight_data), 1) consumer_in_flight_data = parse(consumer_in_flight_data[ctx.msg.msg_id]) self.assertTrue( consumer_in_flight_data < now, 'consumer_in_flight_data:`{}` is not less than now:`{}`'.format( consumer_in_flight_data, now)) # There should still be one unacknowledged message. unack_counter = self.kvdb.hgetall(self.api.impl.UNACK_COUNTER_KEY) self.assertEquals(len(unack_counter), 1) self.assertEqual(unack_counter[ctx.msg.msg_id], '1') # One subscriber hence one undelivered message def test_get_reject_acknowledge(self): payload, topic, producer, ctx = self._publish_move(move=False) client_id, client_name = rand_int(), rand_string() client = Client(client_id, client_name) sub_key = self.api.subscribe(client.id, topic.name) # Moves a message to the consumer's queue self.api.impl.move_to_target_queues() self._check_consumer_queue_before_get(ctx, sub_key) # Consumer gets a message which puts it in the in-flight state. self._check_get(ctx, sub_key, topic, producer, client) # However, there should be nothing in the consumer's queue. consumer_msg_ids = self.kvdb.lrange(self.api.impl.CONSUMER_MSG_IDS_PREFIX.format(sub_key), 0, -1) self.assertEquals(consumer_msg_ids, []) # Consumer rejects the message which puts it back on a queue. self.api.reject(sub_key, ctx.msg.msg_id) # After rejection it's as though the message has just been published. self._check_consumer_queue_before_get(ctx, sub_key) # Get after rejection works as before. self._check_get(ctx, sub_key, topic, producer, client) # Consumer acknowledges a message. self.api.acknowledge(sub_key, ctx.msg.msg_id) # This was the only one subscription so now that the message has been delivered # there should be no trace of it in backend. # The only keys left are LAST_PUB_TIME_KEY, LAST_SEEN_CONSUMER_KEY and LAST_SEEN_PRODUCER_KEY - nothing else. keys = self.kvdb.keys('{}*'.format(self.key_prefix)) self.assertEquals(len(keys), 3) now = datetime.utcnow() last_pub_time = parse(self.kvdb.hgetall(self.api.impl.LAST_PUB_TIME_KEY)[topic.name]) last_seen_consumer = parse(self.kvdb.hgetall(self.api.impl.LAST_SEEN_CONSUMER_KEY)[str(client.id)]) last_seen_producer = parse(self.kvdb.hgetall(self.api.impl.LAST_SEEN_PRODUCER_KEY)[str(producer.id)]) self.assertTrue(last_pub_time < now, 'last_pub_time:`{}` is not less than now:`{}`'.format(last_pub_time, now)) self.assertTrue(last_seen_consumer < now, 'last_seen_consumer:`{}` is not less than now:`{}`'.format(last_seen_consumer, now)) self.assertTrue(last_seen_producer < now, 'last_seen_producer:`{}` is not less than now:`{}`'.format(last_seen_producer, now)) # ################################################################################################################################ def test_pub_sub_exception(self): invalid_sub_key = rand_string() valid_sub_key = rand_string() client_id = rand_int() topic_name = rand_string() consumer = Consumer(client_id, rand_string(), sub_key=valid_sub_key) topic = Topic(topic_name) # Without adding consumer key, validation won't succeed. self.assertRaises(PubSubException, self.api.impl.validate_sub_key, invalid_sub_key) self.assertRaises(PubSubException, self.api.impl.validate_sub_key, valid_sub_key) # After adding a subscription key no error should be raised. self.api.impl.add_consumer(consumer, topic) self.api.impl.add_subscription(valid_sub_key, client_id, topic_name) self.assertRaises(PubSubException, self.api.impl.validate_sub_key, invalid_sub_key) self.api.impl.validate_sub_key(valid_sub_key) # Should not raise any exception now. self.api.impl.delete_consumer(consumer, topic) # After deleting a consumer, validation won't succeed anymore. self.assertRaises(PubSubException, self.api.impl.validate_sub_key, invalid_sub_key) self.assertRaises(PubSubException, self.api.impl.validate_sub_key, valid_sub_key) def invoke_func_sub_key(func, sub_key, *args): list(func(sub_key, *args)) self.assertRaises(PubSubException, invoke_func_sub_key, self.api.get, valid_sub_key) self.assertRaises(PubSubException, invoke_func_sub_key, self.api.get, invalid_sub_key) self.assertRaises(PubSubException, invoke_func_sub_key, self.api.acknowledge, valid_sub_key, 'abc') self.assertRaises(PubSubException, invoke_func_sub_key, self.api.acknowledge, invalid_sub_key, 'def') self.assertRaises(PubSubException, invoke_func_sub_key, self.api.reject, valid_sub_key, 'abc') self.assertRaises(PubSubException, invoke_func_sub_key, self.api.reject, invalid_sub_key, 'def') def test_publish_exceptions(self): payload = rand_string() producer = Client(rand_int(), rand_string()) def invoke_publish(payload, topic, producer_id): self.api.publish(payload, topic, client_id=producer_id) # KeyError because no such producer is in self.api.impl.producers. self.assertRaises(KeyError, invoke_publish, payload, rand_string(), producer.id) # Adding a producer but still, no such topic. self.api.add_producer(producer, Topic(rand_string())) self.assertRaises(PubSubException, invoke_publish, payload, rand_string(), producer.id) # Adding a topic but still PubSubException is raised because the producer is not allowed to use it. topic = Topic(rand_string()) self.api.add_topic(topic) self.assertRaises(PubSubException, invoke_publish, payload, topic.name, producer.id) # Combining the topic and producer, no exception is raised now. self.api.add_producer(producer, topic) invoke_publish(payload, topic.name, producer.id) # But it's not possible to publish to inactive topics. self.api.impl.topics[topic.name].is_active = False self.assertRaises(PubSubException, invoke_publish, payload, topic.name, producer.id) # Make the topic active and it can be published to again. self.api.impl.topics[topic.name].is_active = True invoke_publish(payload, topic.name, producer.id) # Inactive producers cannot publish to topics either. self.api.impl.producers[producer.id].is_active = False self.assertRaises(PubSubException, invoke_publish, payload, topic.name, producer.id) # Making them active means they can publish again. self.api.impl.producers[producer.id].is_active = True invoke_publish(payload, topic.name, producer.id) def test_ping(self): response = self.api.impl.ping() self.assertIsInstance(response, bool) self.assertEquals(response, True) # ################################################################################################################################ def test_default_clients(self): # Initially, default clients are dummy ones. default_consumer = self.api.get_default_consumer() default_producer = self.api.get_default_producer() self.assertEquals(default_consumer.id, None) self.assertEquals(default_consumer.name, None) self.assertEquals(default_consumer.is_active, True) self.assertEquals(default_producer.id, None) self.assertEquals(default_producer.name, None) self.assertEquals(default_producer.is_active, True) cons_id = rand_int() cons_name = rand_string() cons_is_active = rand_bool() prod_name = rand_string() prod_id = rand_int() prod_is_active = rand_bool() cons = Client(cons_id, cons_name, cons_is_active) prod = Client(prod_id, prod_name, prod_is_active) self.api.set_default_consumer(cons) self.api.set_default_producer(prod) default_consumer = self.api.get_default_consumer() default_producer = self.api.get_default_producer() self.assertEquals(default_consumer.id, cons_id) self.assertEquals(default_consumer.name, cons_name) self.assertEquals(default_consumer.is_active, cons_is_active) self.assertEquals(default_producer.id, prod_id) self.assertEquals(default_producer.name, prod_name) self.assertEquals(default_producer.is_active, prod_is_active) # ################################################################################################################################ def test_topic_add(self): name = rand_string() is_active = rand_bool() is_fifo = rand_bool() max_depth = rand_int() topic = Topic(name, is_active, is_fifo, max_depth) self.api.add_topic(topic) self.assertIn(name, self.api.impl.topics) self.assertEquals(len(self.api.impl.topics), 1) given = self.api.impl.topics[name] self.assertEquals(given.name, name) self.assertEquals(given.is_active, is_active) self.assertEquals(given.is_fifo, is_fifo) self.assertEquals(given.max_depth, max_depth) # Adding topic of the same name should not create a new topic because impl.topics is a dictionary self.api.add_topic(topic) self.assertEquals(len(self.api.impl.topics), 1) def test_topic_update(self): self.test_topic_add() # updating a topic works the same like creating it