Ejemplo n.º 1
0
    def fetch_last_known_offsets(self, partitions=None):
        if self.group is None:
            raise ValueError('SimpleClient.group must not be None')

        if partitions is None:
            partitions = self.client.get_partition_ids_for_topic(self.topic)

        responses = self.client.send_offset_fetch_request(
            self.group,
            [OffsetFetchRequestPayload(self.topic, p) for p in partitions],
            fail_on_error=False
        )

        for resp in responses:
            try:
                check_error(resp)
            # API spec says server wont set an error here
            # but 0.8.1.1 does actually...
            except UnknownTopicOrPartitionError:
                pass

            # -1 offset signals no commit is currently stored
            if resp.offset == -1:
                self.offsets[resp.partition] = 0

            # Otherwise we committed the stored offset
            # and need to fetch the next one
            else:
                self.offsets[resp.partition] = resp.offset
Ejemplo n.º 2
0
    def fetch_last_known_offsets(self, partitions=None):
        if self.group is None:
            raise ValueError('SimpleClient.group must not be None')

        if partitions is None:
            partitions = self.client.get_partition_ids_for_topic(self.topic)

        responses = self.client.send_offset_fetch_request(
            self.group,
            [OffsetFetchRequestPayload(self.topic, p) for p in partitions],
            fail_on_error=False)

        for resp in responses:
            try:
                check_error(resp)
            # API spec says server wont set an error here
            # but 0.8.1.1 does actually...
            except UnknownTopicOrPartitionError:
                pass

            # -1 offset signals no commit is currently stored
            if resp.offset == -1:
                self.offsets[resp.partition] = 0

            # Otherwise we committed the stored offset
            # and need to fetch the next one
            else:
                self.offsets[resp.partition] = resp.offset
Ejemplo n.º 3
0
def _check_commit_response_error(resp):
    try:
        check_error(resp)
    except BrokerResponseError as e:
        exception = OffsetCommitError(resp.topic, resp.partition, e.message)
        return exception
    return resp
Ejemplo n.º 4
0
def _check_commit_response_error(resp):
    try:
        check_error(resp)
    except BrokerResponseError as e:
        exception = OffsetCommitError(
            resp.topic,
            resp.partition,
            e.message
        )
        return exception
    return resp
Ejemplo n.º 5
0
def _check_fetch_response_error(resp):
    try:
        check_error(resp)
    except BrokerResponseError:
        # In case of error we set the offset to (-1,)
        return OffsetResponsePayload(
            resp.topic,
            resp.partition,
            resp.error,
            (-1,),
        )
    return resp
Ejemplo n.º 6
0
    def commit_partition_offsets(self, partition_offsets):
        """
        Commit explicit partition/offset pairs.
        """
        self.logger.debug("Committing partition offsets: %s", partition_offsets)

        commit_requests = [
            OffsetCommitRequestPayload(self.consumer.topic, partition, offset, None)
            for partition, offset in partition_offsets.items()
        ]
        commit_responses = self.consumer.client.send_offset_commit_request(self.consumer.group, commit_requests)
        for commit_response in commit_responses:
            check_error(commit_response)
Ejemplo n.º 7
0
    def _update_group_offsets(self):
        logger.info("Consumer fetching stored offsets")
        for partition in self._client.get_partition_ids_for_topic(self._topic):
            (resp, ) = self._client.send_offset_fetch_request(
                self._group_id, [OffsetFetchRequest(self._topic, partition)],
                fail_on_error=False)
            try:
                check_error(resp)
            except UnknownTopicOrPartitionError:
                pass

            if resp.offset == -1:
                self._offsets.commit[partition] = None
            else:
                self._offsets.commit[partition] = resp.offset
Ejemplo n.º 8
0
    def commit_partition_offsets(self, partition_offsets):
        """
        Commit explicit partition/offset pairs.
        """
        self.logger.debug("Committing partition offsets: %s", partition_offsets)

        commit_requests = [
            OffsetCommitRequestPayload(self.consumer.topic, partition, offset, None)
            for partition, offset in partition_offsets.items()
        ]
        commit_responses = self.consumer.client.send_offset_commit_request(
            self.consumer.group,
            commit_requests,
        )
        for commit_response in commit_responses:
            check_error(commit_response)
Ejemplo n.º 9
0
def pluck_topic_offset_or_zero_on_unknown(resp):
    try:
        check_error(resp)
    except UnknownTopicOrPartitionError:
        # If the server doesn't have any commited offsets by this group for
        # this topic, assume it's zero.
        pass
    # The API spec says server wont set an error, but 0.8.1.1 does. The actual
    # check is if the offset is -1.
    if resp.offset == -1:
        return OffsetFetchResponsePayload(
            resp.topic,
            resp.partition,
            0,
            resp.metadata,
            0,
        )
    return resp
Ejemplo n.º 10
0
    def _update_produced_offsets(self):
        """
        Arguments:
            request_time_ms (int): Used to ask for all messages before a
                certain time (ms). There are two special values. Specify -1 to receive the latest
                offset (i.e. the offset of the next coming message) and -2 to receive the earliest
                available offset. Note that because offsets are pulled in descending order, asking for
                the earliest offset will always return you a single element.
        """
        for partition in self._client.get_partition_ids_for_topic(self._topic):
            reqs = [OffsetRequest(self._topic, partition, -1, 1)]

            (resp, ) = self._client.send_offset_request(reqs)

            check_error(resp)
            assert resp.topic == self._topic
            assert resp.partition == partition
            self._offsets.produced[partition] = resp.offsets[0]
Ejemplo n.º 11
0
    def _fetch(self):
        # Create fetch request payloads for all the partitions
        partitions = dict((p, self.buffer_size)
                      for p in self.fetch_offsets.keys())
        while partitions:
            requests = []
            for partition, buffer_size in six.iteritems(partitions):
                requests.append(FetchRequestPayload(self.topic, partition,
                                                    self.fetch_offsets[partition],
                                                    buffer_size))
            # Send request
            responses = self.client.send_fetch_request(
                requests,
                max_wait_time=int(self.fetch_max_wait_time),
                min_bytes=self.fetch_min_bytes,
                fail_on_error=False
            )

            retry_partitions = {}
            for resp in responses:

                try:
                    check_error(resp)
                except UnknownTopicOrPartitionError:
                    log.error('UnknownTopicOrPartitionError for %s:%d',
                              resp.topic, resp.partition)
                    self.client.reset_topic_metadata(resp.topic)
                    raise
                except NotLeaderForPartitionError:
                    log.error('NotLeaderForPartitionError for %s:%d',
                              resp.topic, resp.partition)
                    self.client.reset_topic_metadata(resp.topic)
                    continue
                except OffsetOutOfRangeError:
                    log.warning('OffsetOutOfRangeError for %s:%d. '
                                'Resetting partition offset...',
                                resp.topic, resp.partition)
                    self.reset_partition_offset(resp.partition)
                    # Retry this partition
                    retry_partitions[resp.partition] = partitions[resp.partition]
                    continue
                except FailedPayloadsError as e:
                    log.warning('FailedPayloadsError for %s:%d',
                                e.payload.topic, e.payload.partition)
                    # Retry this partition
                    retry_partitions[e.payload.partition] = partitions[e.payload.partition]
                    continue

                partition = resp.partition
                buffer_size = partitions[partition]

                # Check for partial message
                if resp.messages and isinstance(resp.messages[-1].message, PartialMessage):

                    # If buffer is at max and all we got was a partial message
                    # raise ConsumerFetchSizeTooSmall
                    if (self.max_buffer_size is not None and
                        buffer_size == self.max_buffer_size and
                        len(resp.messages) == 1):

                        log.error('Max fetch size %d too small', self.max_buffer_size)
                        raise ConsumerFetchSizeTooSmall()

                    if self.max_buffer_size is None:
                        buffer_size *= 2
                    else:
                        buffer_size = min(buffer_size * 2, self.max_buffer_size)
                    log.warning('Fetch size too small, increase to %d (2x) '
                                'and retry', buffer_size)
                    retry_partitions[partition] = buffer_size
                    resp.messages.pop()

                for message in resp.messages:
                    if message.offset < self.fetch_offsets[partition]:
                        log.debug('Skipping message %s because its offset is less than the consumer offset',
                                  message)
                        continue
                    # Put the message in our queue
                    self.queue.put((partition, message))
                    self.fetch_offsets[partition] = message.offset + 1
            partitions = retry_partitions