def test_consumer_remembers_offset(self, fetch): """Test that offsets are successfully retrieved from zk. """ topic = 'testtopic' group = 'testgroup' offset = 10 ev = Event() fake_partition = mock.Mock() fake_partition.cluster = self.c fake_partition.topic.name = topic fake_partition.broker.id = 0 fake_partition.number = 0 def fake_fetch(*args, **kwargs): ev.set() return () fetch.side_effect = fake_fetch op = OwnedPartition(fake_partition, group) op._current_offset = offset op.commit_offset() self._register_fake_brokers(1) t = Topic(self.c, topic) c = t.subscribe(group) self.assertEquals(len(c.partitions), 1) p = list(c.partitions)[0] self.assertEquals(p.offset, offset) self.assertEquals(None, p.next_message(0)) ev.wait(1) fetch.assert_called_with(offset, ConsumerConfig.fetch_size) c.stop_partitions()
def test_commits_offsets(self, fetch): """Test that message offsets are persisted to ZK. """ self._register_fake_brokers(1) t = Topic(self.c, 'testtopic') c = t.subscribe('group') msgs = [] for i in xrange(1, 10): msg = mock.Mock() msg.next_offset = 3 * i msg.payload = str(i) * 3 msgs.append(msg) fetch.return_value = msgs self.assertPassesWithMultipleAttempts( lambda: self.assertTrue(c.next_message(10) is not None), 5) self.assertEquals(len(c.partitions), 1) p = list(c.partitions)[0] self.assertEquals(p.offset, 3) c.commit_offsets() d, stat = self.client.get(p.path) self.assertEquals(d, '3') c.stop_partitions()
def test_commits_offsets(self, fetch): """Test that message offsets are persisted to ZK. """ self._register_fake_brokers(1) t = Topic(self.c, 'testtopic') c = t.subscribe('group') msgs = [] for i in xrange(1, 10): msg = mock.Mock() msg.next_offset = 3 * i msg.payload = str(i) * 3 msgs.append(msg) fetch.return_value = msgs self.assertPassesWithMultipleAttempts( lambda: self.assertTrue(c.next_message(10) is not None), 5 ) self.assertEquals(len(c.partitions), 1) p = list(c.partitions)[0] self.assertEquals(p.offset, 3) c.commit_offsets() d, stat = self.client.get(p.path) self.assertEquals(d, '3') c.stop_partitions()
def test_empty_topic(self): """Test that consuming an empty topic returns an empty list. """ topic = 'topic' t = Topic(self.kafka_cluster, topic) consumer = t.subscribe('group2') self.assertTrue(consumer.empty())
def test_empty_topic(self): """Test that consuming an empty topic returns an empty list. """ topic = self.get_topic().name t = Topic(self.kafka_cluster, topic) consumer = t.subscribe('group2') self.assertTrue(consumer.empty()) consumer.stop_partitions()
def test_empty_topic(self): """Test that consuming an empty topic returns an empty list. """ topic = self.get_topic().name t = Topic(self.kafka_cluster, topic) consumer = t.subscribe('group2') self.assertTrue(consumer.empty()) consumer.stop_partitions()
def test_too_many_consumers(self, *args): """Test graceful failure when # of consumers exceeds partitions """ n_partitions = 1 n_consumers = 2 self._register_fake_brokers(n_partitions) t = Topic(self.c, 'testtopic') with self.assertRaises(NoAvailablePartitionsError): consumers = [t.subscribe('group1') for i in xrange(n_consumers)]
def test_too_many_consumers(self, *args): """Test graceful failure when # of consumers exceeds partitions """ n_partitions = 1 n_consumers = 2 self._register_fake_brokers(n_partitions) t = Topic(self.c, 'testtopic') with self.assertRaises(NoAvailablePartitionsError): consumers = [t.subscribe('group1') for i in xrange(n_consumers)]
def test_multiclient_rebalance(self, *args): """Test rebalancing with many connected clients This test is primarily good at ferreting out concurrency bugs and therefore doesn't test one specific thing, since such bugs are fundamentally hard to trap. To test, it simulates 10 consumer connecting over time, rebalancing, and eventually disconnecting one by one. In order to accomplish this, it creates a separate kazoo client for each consumer. This is required because otherwise there is too much thread contention over the single kazoo client. Some callbacks won't happen until much too late because there aren't enough threads to go around. """ n_partitions = 10 n_consumers = 10 t = Topic(self.c, 'testtopic') # with self.client, new clients don't see the brokers -- unsure why from kazoo.client import KazooClient zk_hosts = ','.join( ['%s:%s' % (h, p) for h, p in self.client.hosts.hosts]) zkclient = KazooClient(hosts=zk_hosts) zkclient.start() self._register_fake_brokers(n_partitions, client=zkclient) zkclient.ensure_path('/brokers/topics') # bring up consumers consumers = [] for i in xrange(n_consumers): newclient = KazooClient(hosts=zk_hosts) newclient.start() cluster = Cluster(newclient) topic = Topic(cluster, 'testtopic') consumers.append((newclient, topic.subscribe('group1'))) time.sleep(1) time.sleep(5) # let things settle # bring down consumers for client, consumer in consumers: consumer.stop_partitions() client.stop() time.sleep( 2) # a little more time so we don't kill during rebalance newclient.stop()
def test_multiclient_rebalance(self, *args): """Test rebalancing with many connected clients This test is primarily good at ferreting out concurrency bugs and therefore doesn't test one specific thing, since such bugs are fundamentally hard to trap. To test, it simulates 10 consumer connecting over time, rebalancing, and eventually disconnecting one by one. In order to accomplish this, it creates a separate kazoo client for each consumer. This is required because otherwise there is too much thread contention over the single kazoo client. Some callbacks won't happen until much too late because there aren't enough threads to go around. """ n_partitions = 10 n_consumers = 10 t = Topic(self.c, 'testtopic') # with self.client, new clients don't see the brokers -- unsure why from kazoo.client import KazooClient zk_hosts = ','.join( ['%s:%s' % (h,p) for h,p in self.client.hosts.hosts]) zkclient = KazooClient(hosts=zk_hosts) zkclient.start() self._register_fake_brokers(n_partitions, client=zkclient) zkclient.ensure_path('/brokers/topics') # bring up consumers consumers = [] for i in xrange(n_consumers): newclient = KazooClient(hosts=zk_hosts) newclient.start() cluster = Cluster(newclient) topic = Topic(cluster, 'testtopic') consumers.append((newclient, topic.subscribe('group1'))) time.sleep(1) time.sleep(5) # let things settle # bring down consumers for client,consumer in consumers: consumer.stop_partitions() client.stop() time.sleep(2) # a little more time so we don't kill during rebalance newclient.stop()
def test_consumes(self): """Test that we can consume messages from kafka. """ topic = 'topic' messages = ['hello world', 'foobar'] # publish `messages` to `topic` self.kafka.produce(topic, 0, messages) t = Topic(self.kafka_cluster, topic) # subscribe to `topic` consumer = t.subscribe('group2') def test(): """Test that `consumer` can see `messages`. catches exceptions so we can retry while we wait for kafka to coallesce. """ logger.debug('Running `test`...') try: self.assertEquals( list(islice(consumer, 0, len(messages))), messages ) return True except AssertionError as e: logger.exception('Caught exception: %s', e) return False # wait for one second for :func:`test` to return true or raise an error polling_timeout(test, 1) old_offset = [p.offset for p in consumer.partitions][0] # test that the offset of our 1 partition is not 0 self.assertTrue(old_offset > 0) # and that consumer contains no more messages. self.assertTrue(consumer.empty()) # repeat and see if offset grows. self.kafka.produce(topic, 0, messages) polling_timeout(test, 1) self.assertTrue([p.offset for p in consumer.partitions][0] > old_offset) consumer.stop_partitions()
def test_consumes(self): """Test that we can consume messages from kafka. """ topic = self.get_topic().name messages = ['hello world', 'foobar'] # publish `messages` to topic self.kafka.produce(topic, 0, messages) t = Topic(self.kafka_cluster, topic) # subscribe to topic consumer = t.subscribe('group2') def test(): """Test that `consumer` can see `messages`. catches exceptions so we can retry while we wait for kafka to coallesce. """ logger.debug('Running `test`...') try: self.assertEquals(list(islice(consumer, 0, len(messages))), messages) return True except AssertionError as e: logger.exception('Caught exception: %s', e) return False # wait for one second for :func:`test` to return true or raise an error polling_timeout(test, 1) old_offset = [p.offset for p in consumer.partitions][0] # test that the offset of our 1 partition is not 0 self.assertTrue(old_offset > 0) # and that consumer contains no more messages. self.assertTrue(consumer.empty()) # repeat and see if offset grows. self.kafka.produce(topic, 0, messages) polling_timeout(test, 1) self.assertTrue([p.offset for p in consumer.partitions][0] > old_offset) consumer.stop_partitions()
def test_fetch_invalid_offset(self): """Test that fetching rolled-over offset skips to lowest valid offset. Bad offsets happen when kafka logs are rolled over automatically. This could happen when a consumer is offline for a long time and zookeeper has an old offset stored. It could also happen with a consumer near the end of a log that's being rolled over and its previous spot no longer exists. """ topic = 'topic' messages = ['hello world', 'foobar'] # publish `messages` to `topic` self.kafka.produce(topic, 0, messages) t = Topic(self.kafka_cluster, topic) # get the consumer and set the offset to -1 consumer = t.subscribe('group2') list(consumer.partitions)[0]._next_offset = -1 def test(): """Test that `consumer` can see `messages`. catches exceptions so we can retry while we wait for kafka to coallesce. """ logger.debug('Running `test`...') try: self.assertEquals( list(islice(consumer, 0, len(messages))), messages ) return True except AssertionError as e: logger.exception('Caught exception: %s', e) return False # wait for one second for :func:`test` to return true or raise an error polling_timeout(test, 1) consumer.stop_partitions()
def test_broker_addition(self, rebalance): """Test adding a broker, and ensure all partitions are discovered Testing this makes sure all the relevant zookeeper watches are functioning and rebalancing is happening when partitions or brokers are added or removed """ t = Topic(self.c, 'testtopic') self._register_fake_broker(0, "creator:127.0.0.1:9092") self._register_fake_partitions('testtopic', n_partitions=2) consumer = t.subscribe('group1') self._register_fake_broker(1, "creator:127.0.0.1:9093") self._register_fake_partitions('testtopic', n_partitions=2, brokers=['1']) time.sleep(1) # let watches resolve self.assertEqual(len(t.partitions), 4) self.assertEqual(len(consumer.partitions), 4)
def test_fetch_invalid_offset(self): """Test that fetching rolled-over offset skips to lowest valid offset. Bad offsets happen when kafka logs are rolled over automatically. This could happen when a consumer is offline for a long time and zookeeper has an old offset stored. It could also happen with a consumer near the end of a log that's being rolled over and its previous spot no longer exists. """ topic = self.get_topic().name messages = ['hello world', 'foobar'] # publish `messages` to topic self.kafka.produce(topic, 0, messages) t = Topic(self.kafka_cluster, topic) # get the consumer and set the offset to -1 consumer = t.subscribe('group2') list(consumer.partitions)[0]._next_offset = -1 def test(): """Test that `consumer` can see `messages`. catches exceptions so we can retry while we wait for kafka to coallesce. """ logger.debug('Running `test`...') try: self.assertEquals(list(islice(consumer, 0, len(messages))), messages) return True except AssertionError as e: logger.exception('Caught exception: %s', e) return False # wait for one second for :func:`test` to return true or raise an error polling_timeout(test, 1) consumer.stop_partitions()
def test_assigns_partitions(self, *args): """ Test rebalance Adjust n_* to see how rebalancing performs. """ n_partitions = 10 n_consumers = 3 self._register_fake_brokers(n_partitions) t = Topic(self.c, 'testtopic') consumers = [t.subscribe('group1') for i in xrange(n_consumers)] partitions = [] for c in consumers: partitions.extend(c.partitions) # test that there are no duplicates. self.assertEquals(len(partitions), n_partitions) # test that every partitions is represented. self.assertEquals(len(set(partitions)), n_partitions)
def test_consumer_remembers_offset(self, fetch): """Test that offsets are successfully retrieved from zk. """ return # TODO: Fix this test topic = 'testtopic' group = 'testgroup' offset = 10 ev = Event() fake_partition = mock.Mock() fake_partition.cluster = self.c fake_partition.topic.name = topic fake_partition.broker.id = 0 fake_partition.number = 0 def fake_fetch(*args, **kwargs): ev.set() return () fetch.side_effect = fake_fetch msgqueue = Queue.Queue() op = OwnedPartition(fake_partition, group, msgqueue) op._current_offset = offset op.commit_offset() self._register_fake_brokers(1) t = Topic(self.c, topic) c = t.subscribe(group) self.assertEquals(len(c.partitions), 1) p = list(c.partitions)[0] self.assertEquals(p.offset, offset) self.assertEquals(None, p.next_message(0)) ev.wait(1) fetch.assert_called_with(offset, ConsumerConfig.fetch_size) c.stop_partitions()
def test_broker_addition(self, rebalance): """Test adding a broker, and ensure all partitions are discovered Testing this makes sure all the relevant zookeeper watches are functioning and rebalancing is happening when partitions or brokers are added or removed """ t = Topic(self.c, 'testtopic') self._register_fake_broker(0, "creator:127.0.0.1:9092") self._register_fake_partitions('testtopic', n_partitions=2) consumer = t.subscribe('group1') self._register_fake_broker(1, "creator:127.0.0.1:9093") self._register_fake_partitions('testtopic', n_partitions=2, brokers=['1']) time.sleep(1) # let watches resolve self.assertEqual(len(t.partitions), 4) self.assertEqual(len(consumer.partitions), 4)
def test_assigns_partitions(self, *args): """ Test rebalance Adjust n_* to see how rebalancing performs. """ n_partitions = 10 n_consumers = 3 self._register_fake_brokers(n_partitions) t = Topic(self.c, 'testtopic') consumers = [t.subscribe('group1') for i in xrange(n_consumers)] partitions = [] for c in consumers: partitions.extend(c.partitions) # test that there are no duplicates. self.assertEquals(len(partitions), n_partitions) # test that every partitions is represented. self.assertEquals(len(set(partitions)), n_partitions)