async def seek_limit(self, internal_name: Tuple[str, str], content_type: str, request_data: dict, beginning: bool = True): direction = "beginning" if beginning else "end" self.log.info("Seeking %s offsets", direction) self._assert_consumer_exists(internal_name, content_type) self._assert_has_key(request_data, "partitions", content_type) resets = [] for el in request_data["partitions"]: convert_to_int(el, "partition", content_type) for k in ["topic", "partition"]: self._assert_has_key(el, k, content_type) resets.append( TopicPartition(topic=el["topic"], partition=el["partition"])) async with self.consumer_locks[internal_name]: consumer = self.consumers[internal_name].consumer try: if beginning: consumer.seek_to_beginning(*resets) else: consumer.seek_to_end(*resets) empty_response() except AssertionError: self._illegal_state_fail( f"Trying to reset unassigned partitions to {direction}", content_type)
async def seek_to(self, internal_name: Tuple[str, str], content_type: str, request_data: dict): self.log.info("Resetting offsets for %s to %r", internal_name, request_data) self._assert_consumer_exists(internal_name, content_type) self._assert_has_key(request_data, "offsets", content_type) seeks = [] for el in request_data["offsets"]: self._assert_has_key(el, "topic", content_type) for k in ["offset", "partition"]: self._assert_has_key(el, k, content_type) convert_to_int(el, k, content_type) self._assert_positive_number(el, "offset", content_type) seeks.append( (TopicPartition(topic=el["topic"], partition=el["partition"]), el["offset"])) async with self.consumer_locks[internal_name]: consumer = self.consumers[internal_name].consumer for part, offset in seeks: try: consumer.seek(part, offset) except AssertionError: self._illegal_state_fail(f"Partition {part} is unassigned", content_type) empty_response()
async def delete_subscription(self, internal_name: Tuple[str, str], content_type: str): self.log.info("Deleting subscription for %s", internal_name) self._assert_consumer_exists(internal_name, content_type) async with self.consumer_locks[internal_name]: self.consumers[internal_name].consumer.unsubscribe() empty_response()
async def commit_offsets(self, internal_name: Tuple[str, str], content_type: str, request_data: dict, cluster_metadata: dict): self.log.info("Committing offsets for %s", internal_name) self._assert_consumer_exists(internal_name, content_type) if request_data: self._assert_has_key(request_data, "offsets", content_type) payload = {} for el in request_data.get("offsets", []): for k in ["partition", "offset"]: convert_to_int(el, k, content_type) # If we commit for a partition that does not belong to this consumer, then the internal error raised # is marked as retriable, and thus the commit method will remain blocked in what looks like an infinite loop self._topic_and_partition_valid(cluster_metadata, el, content_type) payload[TopicPartition(el["topic"], el["partition"])] = OffsetAndMetadata( el["offset"] + 1, None) async with self.consumer_locks[internal_name]: consumer = self.consumers[internal_name].consumer payload = payload or None try: consumer.commit(offsets=payload) except KafkaError as e: KarapaceBase.internal_error( message=f"error sending commit request: {e}", content_type=content_type) empty_response()
async def delete_consumer(self, internal_name: Tuple[str, str], content_type: str): self.log.info("Deleting consumer for %s", internal_name) self._assert_consumer_exists(internal_name, content_type) async with self.consumer_locks[internal_name]: try: c = self.consumers.pop(internal_name) c.consumer.close() self.consumer_locks.pop(internal_name) except: # pylint: disable=bare-except self.log.exception("Unable to properly dispose of consumer") finally: empty_response()
async def set_assignments(self, internal_name: Tuple[str, str], content_type: str, request_data: dict): self.log.info("Updating assignments for %s to %r", internal_name, request_data) self._assert_consumer_exists(internal_name, content_type) self._assert_has_key(request_data, "partitions", content_type) partitions = [] for el in request_data["partitions"]: convert_to_int(el, "partition", content_type) self._has_topic_and_partition_keys(el, content_type) partitions.append(TopicPartition(el["topic"], el["partition"])) async with self.consumer_locks[internal_name]: try: consumer = self.consumers[internal_name].consumer consumer.assign(partitions) self._update_partition_assignments(consumer) empty_response() except IllegalStateError as e: self._illegal_state_fail(message=str(e), content_type=content_type) finally: self.log.info("Done updating assignment")
async def set_subscription(self, internal_name: Tuple[str, str], content_type: str, request_data: dict): self.log.info("Updating subscription for %s", internal_name) self._assert_consumer_exists(internal_name, content_type) topics = request_data.get("topics", []) topics_pattern = request_data.get("topic_pattern") async with self.consumer_locks[internal_name]: consumer = self.consumers[internal_name].consumer try: consumer.subscribe(topics=topics, pattern=topics_pattern) self._update_partition_assignments(consumer) empty_response() except AssertionError: self._illegal_state_fail( message="Neither topic_pattern nor topics are present in request", content_type=content_type ) except IllegalStateError as e: self._illegal_state_fail(str(e), content_type=content_type) finally: self.log.info("Done updating subscription")