def create_consumer(
self,
topics,
cluster_name,
group_id,
auto_offset_reset,
queued_max_messages_kbytes,
queued_min_messages,
):
consumer_config = kafka_config.get_kafka_consumer_cluster_options(
cluster_name,
override_params={
"enable.auto.commit": False,
"group.id": group_id,
"default.topic.config": {"auto.offset.reset": auto_offset_reset},
# overridden to reduce memory usage when there's a large backlog
"queued.max.messages.kbytes": queued_max_messages_kbytes,
"queued.min.messages": queued_min_messages,
},
)
if settings.KAFKA_CONSUMER_AUTO_CREATE_TOPICS:
# This is required for confluent-kafka>=1.5.0, otherwise the topics will
# not be automatically created.
conf = kafka_config.get_kafka_admin_cluster_options(
cluster_name, override_params={"allow.auto.create.topics": "true"}
)
admin_client = AdminClient(conf)
wait_for_topics(admin_client, topics)
consumer = Consumer(consumer_config)
return consumer
def __init__(
self,
group_id,
topic=None,
commit_batch_size=100,
initial_offset_reset="earliest",
force_offset_reset=None,
):
self.group_id = group_id
if not topic:
topic = settings.KAFKA_EVENTS_SUBSCRIPTIONS_RESULTS
self.topic = topic
cluster_name = settings.KAFKA_TOPICS[topic]["cluster"]
self.commit_batch_size = commit_batch_size
self.initial_offset_reset = initial_offset_reset
self.offsets = {}
self.consumer = None
self.cluster_options = kafka_config.get_kafka_consumer_cluster_options(
cluster_name,
{
"group.id": self.group_id,
"session.timeout.ms": 6000,
"auto.offset.reset": self.initial_offset_reset,
"enable.auto.commit": "false",
"enable.auto.offset.store": "false",
"enable.partition.eof": "false",
"default.topic.config": {
"auto.offset.reset": self.initial_offset_reset
},
},
)
self.admin_cluster_options = kafka_config.get_kafka_admin_cluster_options(
cluster_name, {"allow.auto.create.topics": "true"})
self.resolve_partition_force_offset = self.offset_reset_name_to_func(
force_offset_reset)
def create_consumer(
self,
topics,
cluster_name,
group_id,
auto_offset_reset,
queued_max_messages_kbytes,
queued_min_messages,
):
consumer_config = kafka_config.get_kafka_consumer_cluster_options(
cluster_name,
override_params={
"enable.auto.commit": False,
"group.id": group_id,
"default.topic.config": {
"auto.offset.reset": auto_offset_reset
},
# overridden to reduce memory usage when there's a large backlog
"queued.max.messages.kbytes": queued_max_messages_kbytes,
"queued.min.messages": queued_min_messages,
},
)
if settings.KAFKA_CONSUMER_AUTO_CREATE_TOPICS:
# This is required for confluent-kafka>=1.5.0, otherwise the topics will
# not be automatically created.
conf = kafka_config.get_kafka_admin_cluster_options(
cluster_name,
override_params={"allow.auto.create.topics": "true"})
admin_client = AdminClient(conf)
wait_for_topics(admin_client, topics)
consumer = Consumer(consumer_config)
def on_partitions_assigned(consumer, partitions):
logger.info("New partitions assigned: %r", partitions)
def on_partitions_revoked(consumer, partitions):
"Reset the current in-memory batch, letting the next consumer take over where we left off."
logger.info("Partitions revoked: %r", partitions)
self._flush(force=True)
consumer.subscribe(topics,
on_assign=on_partitions_assigned,
on_revoke=on_partitions_revoked)
return consumer
def setUp(self):
super().setUp()
self.events_topic = f"events-{uuid.uuid4().hex}"
self.commit_log_topic = f"events-commit-{uuid.uuid4().hex}"
self.override_settings_cm = override_settings(
KAFKA_TOPICS={
"events": {"cluster": "default", "topic": self.events_topic},
"snuba-commit-log": {"cluster": "default", "topic": self.commit_log_topic},
},
)
self.override_settings_cm.__enter__()
cluster_options = kafka_config.get_kafka_admin_cluster_options(
"default", {"allow.auto.create.topics": "true"}
)
self.admin_client = AdminClient(cluster_options)
wait_for_topics(self.admin_client, [self.events_topic, self.commit_log_topic])
def __init__(
self,
group_id: str,
topic: Optional[str] = None,
commit_batch_size: int = 100,
commit_batch_timeout_ms: int = 5000,
initial_offset_reset: str = "earliest",
force_offset_reset: Optional[str] = None,
):
self.group_id = group_id
if not topic:
# TODO(typing): Need a way to get the actual value of settings to avoid this
topic = cast(str, settings.KAFKA_EVENTS_SUBSCRIPTIONS_RESULTS)
self.topic = topic
cluster_name: str = settings.KAFKA_TOPICS[topic]["cluster"]
self.commit_batch_size = commit_batch_size
# Adding time based commit behaviour
self.commit_batch_timeout_ms: int = commit_batch_timeout_ms
self.__batch_deadline: Optional[float] = None
self.initial_offset_reset = initial_offset_reset
self.offsets: Dict[int, Optional[int]] = {}
self.consumer: Consumer = None
self.cluster_options = kafka_config.get_kafka_consumer_cluster_options(
cluster_name,
{
"group.id": self.group_id,
"session.timeout.ms": 6000,
"auto.offset.reset": self.initial_offset_reset,
"enable.auto.commit": "false",
"enable.auto.offset.store": "false",
"enable.partition.eof": "false",
"default.topic.config": {
"auto.offset.reset": self.initial_offset_reset
},
},
)
self.admin_cluster_options = kafka_config.get_kafka_admin_cluster_options(
cluster_name, {"allow.auto.create.topics": "true"})
self.resolve_partition_force_offset = self.offset_reset_name_to_func(
force_offset_reset)
self.__shutdown_requested = False
def setUp(self):
super().setUp()
self.ingest_topic = uuid4().hex
self.snuba_topic = uuid4().hex
self.override_settings_cm = override_settings(
KAFKA_TOPICS={
self.ingest_topic: {
"cluster": "default",
"topic": self.ingest_topic
},
self.snuba_topic: {
"cluster": "default",
"topic": self.snuba_topic
},
},
KAFKA_SNUBA_METRICS=self.snuba_topic,
)
self.override_settings_cm.__enter__()
cluster_options = kafka_config.get_kafka_admin_cluster_options(
"default", {"allow.auto.create.topics": "true"})
self.admin_client = AdminClient(cluster_options)
wait_for_topics(self.admin_client, [self.snuba_topic])
def test_get_kafka_admin_cluster_options():
cluster_options = get_kafka_admin_cluster_options("default")
assert (cluster_options["bootstrap.servers"] ==
settings.KAFKA_CLUSTERS["default"]["common"]["bootstrap.servers"])
