示例#1
0
 def _prepare_partition_offsets(self,
                                topic: Topic,
                                partition_data: Dict[int, int],
                                timestamps: bool = False):
     topic_watermarks = topic.watermarks
     offsets = {}
     for partition_id, consumer_offset in partition_data.items():
         # TODO somehow include this in the returned dictionary
         if partition_id not in topic_watermarks:
             log.warning(
                 f"Found invalid offset! Partition {partition_id} does not exist for topic {topic.name}"
             )
         offsets[partition_id] = {
             "consumer_offset":
             consumer_offset,
             "topic_low_watermark":
             topic_watermarks[partition_id].low,
             "topic_high_watermark":
             topic_watermarks[partition_id].high,
             "consumer_lag":
             topic_watermarks[partition_id].high - consumer_offset,
         }
         if timestamps:
             extended_partition_data = topic.get_partition_data(
                 partition_id)
             if extended_partition_data and extended_partition_data.latest_message_timestamp:
                 offsets[partition_id][
                     "latest_timestamp"] = extended_partition_data.latest_message_timestamp
     return offsets
示例#2
0
    def _prepare_offsets(self,
                         topic: Topic,
                         partition_data: Dict[int, int],
                         timestamps: bool = False):
        extended_partition_data = topic.partition_data
        topic_watermarks = topic.watermarks
        new_consumer_offsets = {
            "consumer_offset": (float("inf"), float("-inf")),
            "topic_low_watermark": (float("inf"), float("-inf")),
            "topic_high_watermark": (float("inf"), float("-inf")),
            "consumer_lag": (float("inf"), float("-inf")),
        }
        for partition_id, consumer_offset in partition_data.items():
            current_offset = consumer_offset
            new_consumer_offsets["consumer_offset"] = self._update_minmax(
                new_consumer_offsets["consumer_offset"], current_offset)
            new_consumer_offsets["topic_low_watermark"] = self._update_minmax(
                new_consumer_offsets["topic_low_watermark"],
                topic_watermarks[partition_id].low)
            new_consumer_offsets["topic_high_watermark"] = self._update_minmax(
                new_consumer_offsets["topic_high_watermark"],
                topic_watermarks[partition_id].high)
            new_consumer_offsets["consumer_lag"] = self._update_minmax(
                new_consumer_offsets["consumer_lag"],
                topic_watermarks[partition_id].high - current_offset)

            if timestamps:
                extended_partition_data = topic.get_partition_data(
                    partition_id)
                if extended_partition_data and extended_partition_data.latest_message_timestamp:
                    if "latest_timestamp" in new_consumer_offsets:
                        new_consumer_offsets[
                            "latest_timestamp"] = self._update_minmax(
                                new_consumer_offsets["latest_timestamp"],
                                extended_partition_data.
                                latest_message_timestamp)
                    else:
                        new_consumer_offsets["latest_timestamp"] = (
                            extended_partition_data.latest_message_timestamp,
                            extended_partition_data.latest_message_timestamp,
                        )
        return new_consumer_offsets