示例#1
0
    def test_consumes(self):
        """Test that we can consume messages from kafka.

        """
        topic = 'topic'
        messages = ['hello world', 'foobar']

        # publish `messages` to `topic`
        self.kafka.produce(topic, 0, messages)

        t = Topic(self.kafka_cluster, topic)

        # subscribe to `topic`
        consumer = t.subscribe('group2')

        def test():
            """Test that `consumer` can see `messages`.

            catches exceptions so we can retry while we wait for kafka to
            coallesce.

            """
            logger.debug('Running `test`...')
            try:
                self.assertEquals(
                    list(islice(consumer, 0, len(messages))),
                    messages
                )
                return True
            except AssertionError as e:
                logger.exception('Caught exception: %s', e)
                return False

        # wait for one second for :func:`test` to return true or raise an error
        polling_timeout(test, 1)

        old_offset = [p.offset for p in consumer.partitions][0]
        # test that the offset of our 1 partition is not 0
        self.assertTrue(old_offset > 0)
        # and that consumer contains no more messages.
        self.assertTrue(consumer.empty())

        # repeat and see if offset grows.
        self.kafka.produce(topic, 0, messages)
        polling_timeout(test, 1)
        self.assertTrue([p.offset for p in consumer.partitions][0] > old_offset)
        consumer.stop_partitions()
示例#2
0
    def test_consumes(self):
        """Test that we can consume messages from kafka.

        """
        topic = self.get_topic().name
        messages = ['hello world', 'foobar']

        # publish `messages` to topic
        self.kafka.produce(topic, 0, messages)

        t = Topic(self.kafka_cluster, topic)

        # subscribe to topic
        consumer = t.subscribe('group2')

        def test():
            """Test that `consumer` can see `messages`.

            catches exceptions so we can retry while we wait for kafka to
            coallesce.

            """
            logger.debug('Running `test`...')
            try:
                self.assertEquals(list(islice(consumer, 0, len(messages))),
                                  messages)
                return True
            except AssertionError as e:
                logger.exception('Caught exception: %s', e)
                return False

        # wait for one second for :func:`test` to return true or raise an error
        polling_timeout(test, 1)

        old_offset = [p.offset for p in consumer.partitions][0]
        # test that the offset of our 1 partition is not 0
        self.assertTrue(old_offset > 0)
        # and that consumer contains no more messages.
        self.assertTrue(consumer.empty())

        # repeat and see if offset grows.
        self.kafka.produce(topic, 0, messages)
        polling_timeout(test, 1)
        self.assertTrue([p.offset
                         for p in consumer.partitions][0] > old_offset)
        consumer.stop_partitions()
示例#3
0
    def test_fetch_invalid_offset(self):
        """Test that fetching rolled-over offset skips to lowest valid offset.

        Bad offsets happen when kafka logs are rolled over automatically.
        This could happen when a consumer is offline for a long time and
        zookeeper has an old offset stored. It could also happen with a
        consumer near the end of a log that's being rolled over and its
        previous spot no longer exists.
        """
        topic = 'topic'
        messages = ['hello world', 'foobar']

        # publish `messages` to `topic`
        self.kafka.produce(topic, 0, messages)

        t = Topic(self.kafka_cluster, topic)

        # get the consumer and set the offset to -1
        consumer = t.subscribe('group2')
        list(consumer.partitions)[0]._next_offset = -1

        def test():
            """Test that `consumer` can see `messages`.

            catches exceptions so we can retry while we wait for kafka to
            coallesce.

            """
            logger.debug('Running `test`...')
            try:
                self.assertEquals(
                    list(islice(consumer, 0, len(messages))),
                    messages
                )
                return True
            except AssertionError as e:
                logger.exception('Caught exception: %s', e)
                return False

        # wait for one second for :func:`test` to return true or raise an error
        polling_timeout(test, 1)
        consumer.stop_partitions()
示例#4
0
    def test_fetch_invalid_offset(self):
        """Test that fetching rolled-over offset skips to lowest valid offset.

        Bad offsets happen when kafka logs are rolled over automatically.
        This could happen when a consumer is offline for a long time and
        zookeeper has an old offset stored. It could also happen with a
        consumer near the end of a log that's being rolled over and its
        previous spot no longer exists.
        """
        topic = self.get_topic().name
        messages = ['hello world', 'foobar']

        # publish `messages` to topic
        self.kafka.produce(topic, 0, messages)

        t = Topic(self.kafka_cluster, topic)

        # get the consumer and set the offset to -1
        consumer = t.subscribe('group2')
        list(consumer.partitions)[0]._next_offset = -1

        def test():
            """Test that `consumer` can see `messages`.

            catches exceptions so we can retry while we wait for kafka to
            coallesce.

            """
            logger.debug('Running `test`...')
            try:
                self.assertEquals(list(islice(consumer, 0, len(messages))),
                                  messages)
                return True
            except AssertionError as e:
                logger.exception('Caught exception: %s', e)
                return False

        # wait for one second for :func:`test` to return true or raise an error
        polling_timeout(test, 1)
        consumer.stop_partitions()