def test_kafka_consumer__blocking(self): TIMEOUT_MS = 500 consumer = self.kafka_consumer(auto_offset_reset='smallest', consumer_timeout_ms=TIMEOUT_MS) # Ask for 5 messages, nothing in queue, block 5 seconds with Timer() as t: with self.assertRaises(ConsumerTimeout): msg = consumer.next() self.assertGreaterEqual(t.interval, TIMEOUT_MS / 1000.0 ) self.send_messages(0, range(0, 10)) # Ask for 5 messages, 10 in queue. Get 5 back, no blocking messages = set() with Timer() as t: for i in range(5): msg = consumer.next() messages.add((msg.partition, msg.offset)) self.assertEqual(len(messages), 5) self.assertLess(t.interval, TIMEOUT_MS / 1000.0 ) # Ask for 10 messages, get 5 back, block 5 seconds messages = set() with Timer() as t: with self.assertRaises(ConsumerTimeout): for i in range(10): msg = consumer.next() messages.add((msg.partition, msg.offset)) self.assertEqual(len(messages), 5) self.assertGreaterEqual(t.interval, TIMEOUT_MS / 1000.0 )
def test_simple_consumer_blocking(self): consumer = self.consumer() # Ask for 5 messages, nothing in queue, block 1 second with Timer() as t: messages = consumer.get_messages(block=True, timeout=1) self.assert_message_count(messages, 0) self.assertGreaterEqual(t.interval, 1) self.send_messages(0, range(0, 5)) self.send_messages(1, range(5, 10)) # Ask for 5 messages, 10 in queue. Get 5 back, no blocking with Timer() as t: messages = consumer.get_messages(count=5, block=True, timeout=3) self.assert_message_count(messages, 5) self.assertLess(t.interval, 3) # Ask for 10 messages, get 5 back, block 1 second with Timer() as t: messages = consumer.get_messages(count=10, block=True, timeout=1) self.assert_message_count(messages, 5) self.assertGreaterEqual(t.interval, 1) # Ask for 10 messages, 5 in queue, ask to block for 1 message or 1 # second, get 5 back, no blocking self.send_messages(0, range(0, 3)) self.send_messages(1, range(3, 5)) with Timer() as t: messages = consumer.get_messages(count=10, block=1, timeout=1) self.assert_message_count(messages, 5) self.assertLessEqual(t.interval, 1) consumer.stop()
def test_multi_process_consumer_blocking(self): consumer = self.consumer(consumer = MultiProcessConsumer) # Ask for 5 messages, No messages in queue, block 5 seconds with Timer() as t: messages = consumer.get_messages(block=True, timeout=5) self.assert_message_count(messages, 0) self.assertGreaterEqual(t.interval, 5) # Send 10 messages self.send_messages(0, range(0, 10)) # Ask for 5 messages, 10 messages in queue, block 0 seconds with Timer() as t: messages = consumer.get_messages(count=5, block=True, timeout=5) self.assert_message_count(messages, 5) self.assertLessEqual(t.interval, 1) # Ask for 10 messages, 5 in queue, block 5 seconds with Timer() as t: messages = consumer.get_messages(count=10, block=True, timeout=5) self.assert_message_count(messages, 5) self.assertGreaterEqual(t.interval, 4.95) consumer.stop()
def test_kafka_consumer__blocking(self): TIMEOUT_MS = 500 consumer = self.kafka_consumer(auto_offset_reset='earliest', consumer_timeout_ms=TIMEOUT_MS) # Manual assignment avoids overhead of consumer group mgmt consumer.unsubscribe() consumer.assign([TopicPartition(self.topic, 0)]) # Ask for 5 messages, nothing in queue, block 500ms with Timer() as t: with self.assertRaises(StopIteration): msg = next(consumer) self.assertGreaterEqual(t.interval, TIMEOUT_MS / 1000.0) self.send_messages(0, range(0, 10)) # Ask for 5 messages, 10 in queue. Get 5 back, no blocking messages = set() with Timer() as t: for i in range(5): msg = next(consumer) messages.add((msg.partition, msg.offset)) self.assertEqual(len(messages), 5) self.assertLess(t.interval, TIMEOUT_MS / 1000.0) # Ask for 10 messages, get 5 back, block 500ms messages = set() with Timer() as t: with self.assertRaises(StopIteration): for i in range(10): msg = next(consumer) messages.add((msg.partition, msg.offset)) self.assertEqual(len(messages), 5) self.assertGreaterEqual(t.interval, TIMEOUT_MS / 1000.0)
def test_timeout(self): def _timeout(*args, **kwargs): timeout = args[1] sleep(timeout) raise socket.timeout with patch.object(socket, "create_connection", side_effect=_timeout): with Timer() as t: with self.assertRaises(ConnectionError): KafkaConnection("nowhere", 1234, 1.0) self.assertGreaterEqual(t.interval, 1.0)
def test_kafka_consumer__blocking(kafka_consumer_factory, topic, send_messages): TIMEOUT_MS = 500 consumer = kafka_consumer_factory(auto_offset_reset='earliest', enable_auto_commit=False, consumer_timeout_ms=TIMEOUT_MS) # Manual assignment avoids overhead of consumer group mgmt consumer.unsubscribe() consumer.assign([TopicPartition(topic, 0)]) # Ask for 5 messages, nothing in queue, block 500ms with Timer() as t: with pytest.raises(StopIteration): msg = next(consumer) assert t.interval >= (TIMEOUT_MS / 1000.0) send_messages(range(0, 10)) # Ask for 5 messages, 10 in queue. Get 5 back, no blocking messages = [] with Timer() as t: for i in range(5): msg = next(consumer) messages.append(msg) assert_message_count(messages, 5) assert t.interval < (TIMEOUT_MS / 1000.0) # Ask for 10 messages, get 5 back, block 500ms messages = [] with Timer() as t: with pytest.raises(StopIteration): for i in range(10): msg = next(consumer) messages.append(msg) assert_message_count(messages, 5) assert t.interval >= (TIMEOUT_MS / 1000.0)
def test_multi_process_consumer_blocking(self): consumer = self.consumer(consumer = MultiProcessConsumer) # Ask for 5 messages, No messages in queue, block 1 second with Timer() as t: messages = consumer.get_messages(block=True, timeout=1) self.assert_message_count(messages, 0) self.assertGreaterEqual(t.interval, 1) # Send 10 messages self.send_messages(0, range(0, 10)) # Ask for 5 messages, 10 messages in queue, block 0 seconds with Timer() as t: messages = consumer.get_messages(count=5, block=True, timeout=5) self.assert_message_count(messages, 5) self.assertLessEqual(t.interval, 1) # Ask for 10 messages, 5 in queue, block 1 second with Timer() as t: messages = consumer.get_messages(count=10, block=True, timeout=1) self.assert_message_count(messages, 5) self.assertGreaterEqual(t.interval, 1) # Ask for 10 messages, 5 in queue, ask to block for 1 message or 1 # second, get at least one back, no blocking self.send_messages(0, range(0, 5)) with Timer() as t: messages = consumer.get_messages(count=10, block=1, timeout=1) received_message_count = len(messages) self.assertGreaterEqual(received_message_count, 1) self.assert_message_count(messages, received_message_count) self.assertLessEqual(t.interval, 1) consumer.stop()
def test_simple_consumer_blocking(self): consumer = self.consumer() # Ask for 5 messages, nothing in queue, block 5 seconds with Timer() as t: messages = consumer.get_messages(block=True, timeout=5) self.assert_message_count(messages, 0) self.assertGreaterEqual(t.interval, 5) self.send_messages(0, range(0, 10)) # Ask for 5 messages, 10 in queue. Get 5 back, no blocking with Timer() as t: messages = consumer.get_messages(count=5, block=True, timeout=5) self.assert_message_count(messages, 5) self.assertLessEqual(t.interval, 1) # Ask for 10 messages, get 5 back, block 5 seconds with Timer() as t: messages = consumer.get_messages(count=10, block=True, timeout=5) self.assert_message_count(messages, 5) self.assertGreaterEqual(t.interval, 5) consumer.stop()