def create_consumer( self, topics, cluster_name, group_id, auto_offset_reset, queued_max_messages_kbytes, queued_min_messages, ): consumer_config = kafka_config.get_kafka_consumer_cluster_options( cluster_name, override_params={ "enable.auto.commit": False, "group.id": group_id, "default.topic.config": {"auto.offset.reset": auto_offset_reset}, # overridden to reduce memory usage when there's a large backlog "queued.max.messages.kbytes": queued_max_messages_kbytes, "queued.min.messages": queued_min_messages, }, ) if settings.KAFKA_CONSUMER_AUTO_CREATE_TOPICS: # This is required for confluent-kafka>=1.5.0, otherwise the topics will # not be automatically created. conf = kafka_config.get_kafka_admin_cluster_options( cluster_name, override_params={"allow.auto.create.topics": "true"} ) admin_client = AdminClient(conf) wait_for_topics(admin_client, topics) consumer = Consumer(consumer_config) return consumer
def __init__( self, group_id, topic=None, commit_batch_size=100, initial_offset_reset="earliest", force_offset_reset=None, ): self.group_id = group_id if not topic: topic = settings.KAFKA_EVENTS_SUBSCRIPTIONS_RESULTS self.topic = topic cluster_name = settings.KAFKA_TOPICS[topic]["cluster"] self.commit_batch_size = commit_batch_size self.initial_offset_reset = initial_offset_reset self.offsets = {} self.consumer = None self.cluster_options = kafka_config.get_kafka_consumer_cluster_options( cluster_name, { "group.id": self.group_id, "session.timeout.ms": 6000, "auto.offset.reset": self.initial_offset_reset, "enable.auto.commit": "false", "enable.auto.offset.store": "false", "enable.partition.eof": "false", "default.topic.config": { "auto.offset.reset": self.initial_offset_reset }, }, ) self.admin_cluster_options = kafka_config.get_kafka_admin_cluster_options( cluster_name, {"allow.auto.create.topics": "true"}) self.resolve_partition_force_offset = self.offset_reset_name_to_func( force_offset_reset)
def create_consumer( self, topics, cluster_name, group_id, auto_offset_reset, queued_max_messages_kbytes, queued_min_messages, ): consumer_config = kafka_config.get_kafka_consumer_cluster_options( cluster_name, override_params={ "enable.auto.commit": False, "group.id": group_id, "default.topic.config": { "auto.offset.reset": auto_offset_reset }, # overridden to reduce memory usage when there's a large backlog "queued.max.messages.kbytes": queued_max_messages_kbytes, "queued.min.messages": queued_min_messages, }, ) if settings.KAFKA_CONSUMER_AUTO_CREATE_TOPICS: # This is required for confluent-kafka>=1.5.0, otherwise the topics will # not be automatically created. conf = kafka_config.get_kafka_admin_cluster_options( cluster_name, override_params={"allow.auto.create.topics": "true"}) admin_client = AdminClient(conf) wait_for_topics(admin_client, topics) consumer = Consumer(consumer_config) def on_partitions_assigned(consumer, partitions): logger.info("New partitions assigned: %r", partitions) def on_partitions_revoked(consumer, partitions): "Reset the current in-memory batch, letting the next consumer take over where we left off." logger.info("Partitions revoked: %r", partitions) self._flush(force=True) consumer.subscribe(topics, on_assign=on_partitions_assigned, on_revoke=on_partitions_revoked) return consumer
def setUp(self): super().setUp() self.events_topic = f"events-{uuid.uuid4().hex}" self.commit_log_topic = f"events-commit-{uuid.uuid4().hex}" self.override_settings_cm = override_settings( KAFKA_TOPICS={ "events": {"cluster": "default", "topic": self.events_topic}, "snuba-commit-log": {"cluster": "default", "topic": self.commit_log_topic}, }, ) self.override_settings_cm.__enter__() cluster_options = kafka_config.get_kafka_admin_cluster_options( "default", {"allow.auto.create.topics": "true"} ) self.admin_client = AdminClient(cluster_options) wait_for_topics(self.admin_client, [self.events_topic, self.commit_log_topic])
def __init__( self, group_id: str, topic: Optional[str] = None, commit_batch_size: int = 100, commit_batch_timeout_ms: int = 5000, initial_offset_reset: str = "earliest", force_offset_reset: Optional[str] = None, ): self.group_id = group_id if not topic: # TODO(typing): Need a way to get the actual value of settings to avoid this topic = cast(str, settings.KAFKA_EVENTS_SUBSCRIPTIONS_RESULTS) self.topic = topic cluster_name: str = settings.KAFKA_TOPICS[topic]["cluster"] self.commit_batch_size = commit_batch_size # Adding time based commit behaviour self.commit_batch_timeout_ms: int = commit_batch_timeout_ms self.__batch_deadline: Optional[float] = None self.initial_offset_reset = initial_offset_reset self.offsets: Dict[int, Optional[int]] = {} self.consumer: Consumer = None self.cluster_options = kafka_config.get_kafka_consumer_cluster_options( cluster_name, { "group.id": self.group_id, "session.timeout.ms": 6000, "auto.offset.reset": self.initial_offset_reset, "enable.auto.commit": "false", "enable.auto.offset.store": "false", "enable.partition.eof": "false", "default.topic.config": { "auto.offset.reset": self.initial_offset_reset }, }, ) self.admin_cluster_options = kafka_config.get_kafka_admin_cluster_options( cluster_name, {"allow.auto.create.topics": "true"}) self.resolve_partition_force_offset = self.offset_reset_name_to_func( force_offset_reset) self.__shutdown_requested = False
def setUp(self): super().setUp() self.ingest_topic = uuid4().hex self.snuba_topic = uuid4().hex self.override_settings_cm = override_settings( KAFKA_TOPICS={ self.ingest_topic: { "cluster": "default", "topic": self.ingest_topic }, self.snuba_topic: { "cluster": "default", "topic": self.snuba_topic }, }, KAFKA_SNUBA_METRICS=self.snuba_topic, ) self.override_settings_cm.__enter__() cluster_options = kafka_config.get_kafka_admin_cluster_options( "default", {"allow.auto.create.topics": "true"}) self.admin_client = AdminClient(cluster_options) wait_for_topics(self.admin_client, [self.snuba_topic])
def test_get_kafka_admin_cluster_options(): cluster_options = get_kafka_admin_cluster_options("default") assert (cluster_options["bootstrap.servers"] == settings.KAFKA_CLUSTERS["default"]["common"]["bootstrap.servers"])