def _prepare_partition_offsets(self,
topic: Topic,
partition_data: Dict[int, int],
timestamps: bool = False):
topic_watermarks = topic.watermarks
offsets = {}
for partition_id, consumer_offset in partition_data.items():
# TODO somehow include this in the returned dictionary
if partition_id not in topic_watermarks:
log.warning(
f"Found invalid offset! Partition {partition_id} does not exist for topic {topic.name}"
)
offsets[partition_id] = {
"consumer_offset":
consumer_offset,
"topic_low_watermark":
topic_watermarks[partition_id].low,
"topic_high_watermark":
topic_watermarks[partition_id].high,
"consumer_lag":
topic_watermarks[partition_id].high - consumer_offset,
}
if timestamps:
extended_partition_data = topic.get_partition_data(
partition_id)
if extended_partition_data and extended_partition_data.latest_message_timestamp:
offsets[partition_id][
"latest_timestamp"] = extended_partition_data.latest_message_timestamp
return offsets
def _prepare_offsets(self,
topic: Topic,
partition_data: Dict[int, int],
timestamps: bool = False):
extended_partition_data = topic.partition_data
topic_watermarks = topic.watermarks
new_consumer_offsets = {
"consumer_offset": (float("inf"), float("-inf")),
"topic_low_watermark": (float("inf"), float("-inf")),
"topic_high_watermark": (float("inf"), float("-inf")),
"consumer_lag": (float("inf"), float("-inf")),
}
for partition_id, consumer_offset in partition_data.items():
current_offset = consumer_offset
new_consumer_offsets["consumer_offset"] = self._update_minmax(
new_consumer_offsets["consumer_offset"], current_offset)
new_consumer_offsets["topic_low_watermark"] = self._update_minmax(
new_consumer_offsets["topic_low_watermark"],
topic_watermarks[partition_id].low)
new_consumer_offsets["topic_high_watermark"] = self._update_minmax(
new_consumer_offsets["topic_high_watermark"],
topic_watermarks[partition_id].high)
new_consumer_offsets["consumer_lag"] = self._update_minmax(
new_consumer_offsets["consumer_lag"],
topic_watermarks[partition_id].high - current_offset)
if timestamps:
extended_partition_data = topic.get_partition_data(
partition_id)
if extended_partition_data and extended_partition_data.latest_message_timestamp:
if "latest_timestamp" in new_consumer_offsets:
new_consumer_offsets[
"latest_timestamp"] = self._update_minmax(
new_consumer_offsets["latest_timestamp"],
extended_partition_data.
latest_message_timestamp)
else:
new_consumer_offsets["latest_timestamp"] = (
extended_partition_data.latest_message_timestamp,
extended_partition_data.latest_message_timestamp,
)
return new_consumer_offsets
