def test_kafka_consumer__blocking(self):
TIMEOUT_MS = 500
consumer = self.kafka_consumer(auto_offset_reset='smallest',
consumer_timeout_ms=TIMEOUT_MS)
# Ask for 5 messages, nothing in queue, block 5 seconds
with Timer() as t:
with self.assertRaises(ConsumerTimeout):
msg = consumer.next()
self.assertGreaterEqual(t.interval, TIMEOUT_MS / 1000.0 )
self.send_messages(0, range(0, 10))
# Ask for 5 messages, 10 in queue. Get 5 back, no blocking
messages = set()
with Timer() as t:
for i in range(5):
msg = consumer.next()
messages.add((msg.partition, msg.offset))
self.assertEqual(len(messages), 5)
self.assertLess(t.interval, TIMEOUT_MS / 1000.0 )
# Ask for 10 messages, get 5 back, block 5 seconds
messages = set()
with Timer() as t:
with self.assertRaises(ConsumerTimeout):
for i in range(10):
msg = consumer.next()
messages.add((msg.partition, msg.offset))
self.assertEqual(len(messages), 5)
self.assertGreaterEqual(t.interval, TIMEOUT_MS / 1000.0 )
def test_simple_consumer_blocking(self):
consumer = self.consumer()
# Ask for 5 messages, nothing in queue, block 1 second
with Timer() as t:
messages = consumer.get_messages(block=True, timeout=1)
self.assert_message_count(messages, 0)
self.assertGreaterEqual(t.interval, 1)
self.send_messages(0, range(0, 5))
self.send_messages(1, range(5, 10))
# Ask for 5 messages, 10 in queue. Get 5 back, no blocking
with Timer() as t:
messages = consumer.get_messages(count=5, block=True, timeout=3)
self.assert_message_count(messages, 5)
self.assertLess(t.interval, 3)
# Ask for 10 messages, get 5 back, block 1 second
with Timer() as t:
messages = consumer.get_messages(count=10, block=True, timeout=1)
self.assert_message_count(messages, 5)
self.assertGreaterEqual(t.interval, 1)
# Ask for 10 messages, 5 in queue, ask to block for 1 message or 1
# second, get 5 back, no blocking
self.send_messages(0, range(0, 3))
self.send_messages(1, range(3, 5))
with Timer() as t:
messages = consumer.get_messages(count=10, block=1, timeout=1)
self.assert_message_count(messages, 5)
self.assertLessEqual(t.interval, 1)
consumer.stop()
def test_multi_process_consumer_blocking(self):
consumer = self.consumer(consumer = MultiProcessConsumer)
# Ask for 5 messages, No messages in queue, block 5 seconds
with Timer() as t:
messages = consumer.get_messages(block=True, timeout=5)
self.assert_message_count(messages, 0)
self.assertGreaterEqual(t.interval, 5)
# Send 10 messages
self.send_messages(0, range(0, 10))
# Ask for 5 messages, 10 messages in queue, block 0 seconds
with Timer() as t:
messages = consumer.get_messages(count=5, block=True, timeout=5)
self.assert_message_count(messages, 5)
self.assertLessEqual(t.interval, 1)
# Ask for 10 messages, 5 in queue, block 5 seconds
with Timer() as t:
messages = consumer.get_messages(count=10, block=True, timeout=5)
self.assert_message_count(messages, 5)
self.assertGreaterEqual(t.interval, 4.95)
consumer.stop()
def test_kafka_consumer__blocking(self):
TIMEOUT_MS = 500
consumer = self.kafka_consumer(auto_offset_reset='earliest',
consumer_timeout_ms=TIMEOUT_MS)
# Manual assignment avoids overhead of consumer group mgmt
consumer.unsubscribe()
consumer.assign([TopicPartition(self.topic, 0)])
# Ask for 5 messages, nothing in queue, block 500ms
with Timer() as t:
with self.assertRaises(StopIteration):
msg = next(consumer)
self.assertGreaterEqual(t.interval, TIMEOUT_MS / 1000.0)
self.send_messages(0, range(0, 10))
# Ask for 5 messages, 10 in queue. Get 5 back, no blocking
messages = set()
with Timer() as t:
for i in range(5):
msg = next(consumer)
messages.add((msg.partition, msg.offset))
self.assertEqual(len(messages), 5)
self.assertLess(t.interval, TIMEOUT_MS / 1000.0)
# Ask for 10 messages, get 5 back, block 500ms
messages = set()
with Timer() as t:
with self.assertRaises(StopIteration):
for i in range(10):
msg = next(consumer)
messages.add((msg.partition, msg.offset))
self.assertEqual(len(messages), 5)
self.assertGreaterEqual(t.interval, TIMEOUT_MS / 1000.0)
def test_timeout(self):
def _timeout(*args, **kwargs):
timeout = args[1]
sleep(timeout)
raise socket.timeout
with patch.object(socket, "create_connection", side_effect=_timeout):
with Timer() as t:
with self.assertRaises(ConnectionError):
KafkaConnection("nowhere", 1234, 1.0)
self.assertGreaterEqual(t.interval, 1.0)
def test_kafka_consumer__blocking(kafka_consumer_factory, topic,
send_messages):
TIMEOUT_MS = 500
consumer = kafka_consumer_factory(auto_offset_reset='earliest',
enable_auto_commit=False,
consumer_timeout_ms=TIMEOUT_MS)
# Manual assignment avoids overhead of consumer group mgmt
consumer.unsubscribe()
consumer.assign([TopicPartition(topic, 0)])
# Ask for 5 messages, nothing in queue, block 500ms
with Timer() as t:
with pytest.raises(StopIteration):
msg = next(consumer)
assert t.interval >= (TIMEOUT_MS / 1000.0)
send_messages(range(0, 10))
# Ask for 5 messages, 10 in queue. Get 5 back, no blocking
messages = []
with Timer() as t:
for i in range(5):
msg = next(consumer)
messages.append(msg)
assert_message_count(messages, 5)
assert t.interval < (TIMEOUT_MS / 1000.0)
# Ask for 10 messages, get 5 back, block 500ms
messages = []
with Timer() as t:
with pytest.raises(StopIteration):
for i in range(10):
msg = next(consumer)
messages.append(msg)
assert_message_count(messages, 5)
assert t.interval >= (TIMEOUT_MS / 1000.0)
def test_multi_process_consumer_blocking(self):
consumer = self.consumer(consumer = MultiProcessConsumer)
# Ask for 5 messages, No messages in queue, block 1 second
with Timer() as t:
messages = consumer.get_messages(block=True, timeout=1)
self.assert_message_count(messages, 0)
self.assertGreaterEqual(t.interval, 1)
# Send 10 messages
self.send_messages(0, range(0, 10))
# Ask for 5 messages, 10 messages in queue, block 0 seconds
with Timer() as t:
messages = consumer.get_messages(count=5, block=True, timeout=5)
self.assert_message_count(messages, 5)
self.assertLessEqual(t.interval, 1)
# Ask for 10 messages, 5 in queue, block 1 second
with Timer() as t:
messages = consumer.get_messages(count=10, block=True, timeout=1)
self.assert_message_count(messages, 5)
self.assertGreaterEqual(t.interval, 1)
# Ask for 10 messages, 5 in queue, ask to block for 1 message or 1
# second, get at least one back, no blocking
self.send_messages(0, range(0, 5))
with Timer() as t:
messages = consumer.get_messages(count=10, block=1, timeout=1)
received_message_count = len(messages)
self.assertGreaterEqual(received_message_count, 1)
self.assert_message_count(messages, received_message_count)
self.assertLessEqual(t.interval, 1)
consumer.stop()
def test_simple_consumer_blocking(self):
consumer = self.consumer()
# Ask for 5 messages, nothing in queue, block 5 seconds
with Timer() as t:
messages = consumer.get_messages(block=True, timeout=5)
self.assert_message_count(messages, 0)
self.assertGreaterEqual(t.interval, 5)
self.send_messages(0, range(0, 10))
# Ask for 5 messages, 10 in queue. Get 5 back, no blocking
with Timer() as t:
messages = consumer.get_messages(count=5, block=True, timeout=5)
self.assert_message_count(messages, 5)
self.assertLessEqual(t.interval, 1)
# Ask for 10 messages, get 5 back, block 5 seconds
with Timer() as t:
messages = consumer.get_messages(count=10, block=True, timeout=5)
self.assert_message_count(messages, 5)
self.assertGreaterEqual(t.interval, 5)
consumer.stop()
