def test_consumes(self):
"""Test that we can consume messages from kafka.
"""
topic = 'topic'
messages = ['hello world', 'foobar']
# publish `messages` to `topic`
self.kafka.produce(topic, 0, messages)
t = Topic(self.kafka_cluster, topic)
# subscribe to `topic`
consumer = t.subscribe('group2')
def test():
"""Test that `consumer` can see `messages`.
catches exceptions so we can retry while we wait for kafka to
coallesce.
"""
logger.debug('Running `test`...')
try:
self.assertEquals(
list(islice(consumer, 0, len(messages))),
messages
)
return True
except AssertionError as e:
logger.exception('Caught exception: %s', e)
return False
# wait for one second for :func:`test` to return true or raise an error
polling_timeout(test, 1)
old_offset = [p.offset for p in consumer.partitions][0]
# test that the offset of our 1 partition is not 0
self.assertTrue(old_offset > 0)
# and that consumer contains no more messages.
self.assertTrue(consumer.empty())
# repeat and see if offset grows.
self.kafka.produce(topic, 0, messages)
polling_timeout(test, 1)
self.assertTrue([p.offset for p in consumer.partitions][0] > old_offset)
consumer.stop_partitions()
def test_consumes(self):
"""Test that we can consume messages from kafka.
"""
topic = self.get_topic().name
messages = ['hello world', 'foobar']
# publish `messages` to topic
self.kafka.produce(topic, 0, messages)
t = Topic(self.kafka_cluster, topic)
# subscribe to topic
consumer = t.subscribe('group2')
def test():
"""Test that `consumer` can see `messages`.
catches exceptions so we can retry while we wait for kafka to
coallesce.
"""
logger.debug('Running `test`...')
try:
self.assertEquals(list(islice(consumer, 0, len(messages))),
messages)
return True
except AssertionError as e:
logger.exception('Caught exception: %s', e)
return False
# wait for one second for :func:`test` to return true or raise an error
polling_timeout(test, 1)
old_offset = [p.offset for p in consumer.partitions][0]
# test that the offset of our 1 partition is not 0
self.assertTrue(old_offset > 0)
# and that consumer contains no more messages.
self.assertTrue(consumer.empty())
# repeat and see if offset grows.
self.kafka.produce(topic, 0, messages)
polling_timeout(test, 1)
self.assertTrue([p.offset
for p in consumer.partitions][0] > old_offset)
consumer.stop_partitions()
def test_fetch_invalid_offset(self):
"""Test that fetching rolled-over offset skips to lowest valid offset.
Bad offsets happen when kafka logs are rolled over automatically.
This could happen when a consumer is offline for a long time and
zookeeper has an old offset stored. It could also happen with a
consumer near the end of a log that's being rolled over and its
previous spot no longer exists.
"""
topic = 'topic'
messages = ['hello world', 'foobar']
# publish `messages` to `topic`
self.kafka.produce(topic, 0, messages)
t = Topic(self.kafka_cluster, topic)
# get the consumer and set the offset to -1
consumer = t.subscribe('group2')
list(consumer.partitions)[0]._next_offset = -1
def test():
"""Test that `consumer` can see `messages`.
catches exceptions so we can retry while we wait for kafka to
coallesce.
"""
logger.debug('Running `test`...')
try:
self.assertEquals(
list(islice(consumer, 0, len(messages))),
messages
)
return True
except AssertionError as e:
logger.exception('Caught exception: %s', e)
return False
# wait for one second for :func:`test` to return true or raise an error
polling_timeout(test, 1)
consumer.stop_partitions()
def test_fetch_invalid_offset(self):
"""Test that fetching rolled-over offset skips to lowest valid offset.
Bad offsets happen when kafka logs are rolled over automatically.
This could happen when a consumer is offline for a long time and
zookeeper has an old offset stored. It could also happen with a
consumer near the end of a log that's being rolled over and its
previous spot no longer exists.
"""
topic = self.get_topic().name
messages = ['hello world', 'foobar']
# publish `messages` to topic
self.kafka.produce(topic, 0, messages)
t = Topic(self.kafka_cluster, topic)
# get the consumer and set the offset to -1
consumer = t.subscribe('group2')
list(consumer.partitions)[0]._next_offset = -1
def test():
"""Test that `consumer` can see `messages`.
catches exceptions so we can retry while we wait for kafka to
coallesce.
"""
logger.debug('Running `test`...')
try:
self.assertEquals(list(islice(consumer, 0, len(messages))),
messages)
return True
except AssertionError as e:
logger.exception('Caught exception: %s', e)
return False
# wait for one second for :func:`test` to return true or raise an error
polling_timeout(test, 1)
consumer.stop_partitions()