예제 #1
0
    def _partition(self):
        """Consume messages from kafka

           Consume messages from kafka using the Kazoo SetPartitioner to
           allow multiple consumer processes to negotiate access to the kafka
           partitions
        """

        # KazooClient and SetPartitioner objects need to be instantiated after
        # the consumer process has forked.  Instantiating prior to forking
        # gives the appearance that things are working but after forking the
        # connection to zookeeper is lost and no state changes are visible

        if not self._kazoo_client:
            self._kazoo_client = KazooClient(hosts=self._zookeeper_url)
            self._kazoo_client.start()

            state_change_event = threading.Event()

            self._set_partitioner = (SetPartitioner(
                self._kazoo_client,
                path=self._zookeeper_path,
                set=self._consumer.fetch_offsets.keys(),
                state_change_event=state_change_event,
                identifier=str(datetime.datetime.now())))

        try:
            while 1:
                if self._set_partitioner.failed:
                    raise Exception("Failed to acquire partition")

                elif self._set_partitioner.release:
                    log.info("Releasing locks on partition set {} "
                             "for topic {}".format(self._partitions,
                                                   self._kafka_topic))
                    self._set_partitioner.release_set()

                    self._partitions = []

                elif self._set_partitioner.acquired:
                    if not self._partitions:
                        self._partitions = [p for p in self._set_partitioner]

                        if not self._partitions:
                            log.info("Not assigned any partitions on topic {},"
                                     " waiting for a Partitioner state change".
                                     format(self._kafka_topic))
                            state_change_event.wait()
                            state_change_event.clear()
                            continue

                        log.info("Acquired locks on partition set {} "
                                 "for topic {}".format(self._partitions,
                                                       self._kafka_topic))

                        # Reconstruct the kafka consumer object because the
                        # consumer has no API that allows the set of partitons
                        # to be updated outside of construction.
                        self._consumer.stop()
                        self._consumer = self._create_kafka_consumer(
                            self._partitions)
                        return

                elif self._set_partitioner.allocating:
                    log.info("Waiting to acquire locks on partition set")
                    self._set_partitioner.wait_for_acquire()

        except Exception:
            log.exception('KafkaConsumer encountered fatal exception '
                          'processing messages.')
            raise
    def __iter__(self):
        """Consume messages from kafka using the Kazoo SetPartitioner to
           allow multiple consumer processes to negotiate access to the kafka
           partitions
        """

        # KazooClient and SetPartitioner objects need to be instantiated after
        # the consumer process has forked.  Instantiating prior to forking
        # gives the appearance that things are working but after forking the
        # connection to zookeeper is lost and no state changes are visible

        kazoo_client = KazooClient(hosts=self._zookeeper_url)
        kazoo_client.start()

        set_partitioner = (SetPartitioner(
            kazoo_client,
            path=self._zookeeper_path,
            set=self._consumer.fetch_offsets.keys()))

        consumed_from_kafka = self._statsd.get_counter(
            name='consumed_from_kafka')

        try:
            partitions = []

            while 1:
                if set_partitioner.failed:
                    raise Exception("Failed to acquire partition")

                elif set_partitioner.release:
                    log.info("Releasing locks on partition set {} "
                             "for topic {}".format(partitions,
                                                   self._kafka_topic))
                    set_partitioner.release_set()

                    partitions = []

                elif set_partitioner.acquired:
                    if not partitions:
                        partitions = [p for p in set_partitioner]

                        log.info("Acquired locks on partition set {} "
                                 "for topic {}".format(partitions,
                                                       self._kafka_topic))

                        # Refresh the last known offsets again to make sure
                        # that they are the latest after having acquired the
                        # lock. Updates self._consumer.fetch_offsets.
                        self._consumer.fetch_last_known_offsets()

                        # Modify self._consumer.fetch_offsets to hold only the
                        # offsets for the set of Kafka partitions acquired
                        # by this instance of the persister.

                        partitioned_fetch_offsets = {}
                        for p in partitions:
                            partitioned_fetch_offsets[p] = (
                                self._consumer.fetch_offsets[p])

                        self._consumer.fetch_offsets = partitioned_fetch_offsets

                    # When Kafka resizes the partitions it's possible that it
                    # will remove data at our current offset.  When this
                    # happens the next attempt to read from Kafka will generate
                    # an OffsetOutOfRangeError.  We trap this error and seek to
                    # the head of the current Kafka data.  Because this error
                    # only happens when Kafka removes data we're currently
                    # pointing at we're gauranteed that we won't read any
                    # duplicate data however we will lose any information
                    # between our current offset and the new Kafka head.

                    try:
                        for message in self._consumer:
                            if not set_partitioner.acquired:
                                break
                            consumed_from_kafka += 1

                            log.debug("Consuming message from kafka, "
                                      "partition {}, offset {}".format(
                                          message[0], message[1].offset))

                            yield message
                    except kafka.common.OffsetOutOfRangeError:
                        log.error("Kafka OffsetOutOfRange.  Jumping to head.")
                        self._consumer.seek(0, 0)

                elif set_partitioner.allocating:
                    log.info("Waiting to acquire locks on partition set")
                    set_partitioner.wait_for_acquire()

        except:
            log.exception('KafkaConsumer encountered fatal exception '
                          'processing messages.')
            raise