async def seek_limit(self,
internal_name: Tuple[str, str],
content_type: str,
request_data: dict,
beginning: bool = True):
direction = "beginning" if beginning else "end"
self.log.info("Seeking %s offsets", direction)
self._assert_consumer_exists(internal_name, content_type)
self._assert_has_key(request_data, "partitions", content_type)
resets = []
for el in request_data["partitions"]:
convert_to_int(el, "partition", content_type)
for k in ["topic", "partition"]:
self._assert_has_key(el, k, content_type)
resets.append(
TopicPartition(topic=el["topic"], partition=el["partition"]))
async with self.consumer_locks[internal_name]:
consumer = self.consumers[internal_name].consumer
try:
if beginning:
consumer.seek_to_beginning(*resets)
else:
consumer.seek_to_end(*resets)
empty_response()
except AssertionError:
self._illegal_state_fail(
f"Trying to reset unassigned partitions to {direction}",
content_type)
async def seek_to(self, internal_name: Tuple[str, str], content_type: str,
request_data: dict):
self.log.info("Resetting offsets for %s to %r", internal_name,
request_data)
self._assert_consumer_exists(internal_name, content_type)
self._assert_has_key(request_data, "offsets", content_type)
seeks = []
for el in request_data["offsets"]:
self._assert_has_key(el, "topic", content_type)
for k in ["offset", "partition"]:
self._assert_has_key(el, k, content_type)
convert_to_int(el, k, content_type)
self._assert_positive_number(el, "offset", content_type)
seeks.append(
(TopicPartition(topic=el["topic"],
partition=el["partition"]), el["offset"]))
async with self.consumer_locks[internal_name]:
consumer = self.consumers[internal_name].consumer
for part, offset in seeks:
try:
consumer.seek(part, offset)
except AssertionError:
self._illegal_state_fail(f"Partition {part} is unassigned",
content_type)
empty_response()
async def delete_subscription(self, internal_name: Tuple[str, str],
content_type: str):
self.log.info("Deleting subscription for %s", internal_name)
self._assert_consumer_exists(internal_name, content_type)
async with self.consumer_locks[internal_name]:
self.consumers[internal_name].consumer.unsubscribe()
empty_response()
async def commit_offsets(self, internal_name: Tuple[str, str],
content_type: str, request_data: dict,
cluster_metadata: dict):
self.log.info("Committing offsets for %s", internal_name)
self._assert_consumer_exists(internal_name, content_type)
if request_data:
self._assert_has_key(request_data, "offsets", content_type)
payload = {}
for el in request_data.get("offsets", []):
for k in ["partition", "offset"]:
convert_to_int(el, k, content_type)
# If we commit for a partition that does not belong to this consumer, then the internal error raised
# is marked as retriable, and thus the commit method will remain blocked in what looks like an infinite loop
self._topic_and_partition_valid(cluster_metadata, el, content_type)
payload[TopicPartition(el["topic"],
el["partition"])] = OffsetAndMetadata(
el["offset"] + 1, None)
async with self.consumer_locks[internal_name]:
consumer = self.consumers[internal_name].consumer
payload = payload or None
try:
consumer.commit(offsets=payload)
except KafkaError as e:
KarapaceBase.internal_error(
message=f"error sending commit request: {e}",
content_type=content_type)
empty_response()
async def delete_consumer(self, internal_name: Tuple[str, str], content_type: str):
self.log.info("Deleting consumer for %s", internal_name)
self._assert_consumer_exists(internal_name, content_type)
async with self.consumer_locks[internal_name]:
try:
c = self.consumers.pop(internal_name)
c.consumer.close()
self.consumer_locks.pop(internal_name)
except: # pylint: disable=bare-except
self.log.exception("Unable to properly dispose of consumer")
finally:
empty_response()
async def set_assignments(self, internal_name: Tuple[str, str], content_type: str, request_data: dict):
self.log.info("Updating assignments for %s to %r", internal_name, request_data)
self._assert_consumer_exists(internal_name, content_type)
self._assert_has_key(request_data, "partitions", content_type)
partitions = []
for el in request_data["partitions"]:
convert_to_int(el, "partition", content_type)
self._has_topic_and_partition_keys(el, content_type)
partitions.append(TopicPartition(el["topic"], el["partition"]))
async with self.consumer_locks[internal_name]:
try:
consumer = self.consumers[internal_name].consumer
consumer.assign(partitions)
self._update_partition_assignments(consumer)
empty_response()
except IllegalStateError as e:
self._illegal_state_fail(message=str(e), content_type=content_type)
finally:
self.log.info("Done updating assignment")
async def set_subscription(self, internal_name: Tuple[str, str], content_type: str, request_data: dict):
self.log.info("Updating subscription for %s", internal_name)
self._assert_consumer_exists(internal_name, content_type)
topics = request_data.get("topics", [])
topics_pattern = request_data.get("topic_pattern")
async with self.consumer_locks[internal_name]:
consumer = self.consumers[internal_name].consumer
try:
consumer.subscribe(topics=topics, pattern=topics_pattern)
self._update_partition_assignments(consumer)
empty_response()
except AssertionError:
self._illegal_state_fail(
message="Neither topic_pattern nor topics are present in request", content_type=content_type
)
except IllegalStateError as e:
self._illegal_state_fail(str(e), content_type=content_type)
finally:
self.log.info("Done updating subscription")
