def create_batching_kafka_consumer(topic_names, worker, **options):
cluster_names = set(settings.KAFKA_TOPICS[topic_name]["cluster"] for topic_name in topic_names)
if len(cluster_names) > 1:
raise ValueError(
"Cannot launch Kafka consumer listening to multiple topics ({}) on different clusters ({})".format(
topic_names, cluster_names
)
)
(cluster_name,) = cluster_names
consumer = BatchingKafkaConsumer(
topics=topic_names,
cluster_name=cluster_name,
worker=worker,
metrics=metrics,
metrics_default_tags={
"topics": ",".join(sorted(topic_names)),
"group_id": options.get("group_id"),
},
**options
)
def handler(signum, frame):
consumer.signal_shutdown()
signal.signal(signal.SIGINT, handler)
signal.signal(signal.SIGTERM, handler)
return consumer
def _build_consumer(
self,
consumer_group,
commit_log_topic,
synchronize_commit_group,
commit_batch_size=100,
commit_batch_timeout_ms=5000,
initial_offset_reset="latest",
):
cluster_name = settings.KAFKA_TOPICS[settings.KAFKA_EVENTS]["cluster"]
synchronized_consumer = SynchronizedConsumer(
cluster_name=cluster_name,
consumer_group=consumer_group,
commit_log_topic=commit_log_topic,
synchronize_commit_group=synchronize_commit_group,
initial_offset_reset=initial_offset_reset,
)
concurrency = options.get(_CONCURRENCY_OPTION)
worker = PostProcessForwarderWorker(concurrency=concurrency)
consumer = BatchingKafkaConsumer(
topics=self.topic,
worker=worker,
max_batch_size=commit_batch_size,
max_batch_time=commit_batch_timeout_ms,
consumer=synchronized_consumer,
commit_on_shutdown=True,
)
return consumer
def create_batching_kafka_consumer(topic_names, worker, **options):
cluster_names = set(settings.KAFKA_TOPICS[topic_name]["cluster"]
for topic_name in topic_names)
if len(cluster_names) > 1:
raise ValueError(
"Cannot launch Kafka consumer listening to multiple topics ({}) on different clusters ({})"
.format(topic_names, cluster_names))
cluster_name, = cluster_names
bootstrap_servers = settings.KAFKA_CLUSTERS[cluster_name][
"bootstrap.servers"]
if not isinstance(bootstrap_servers, (list, tuple)):
bootstrap_servers = bootstrap_servers.split(",")
consumer = BatchingKafkaConsumer(topics=topic_names,
bootstrap_servers=bootstrap_servers,
worker=worker,
metrics=metrics,
metrics_default_tags={
"topics":
",".join(sorted(topic_names)),
"group_id": options.get("group_id"),
},
**options)
def handler(signum, frame):
consumer.signal_shutdown()
signal.signal(signal.SIGINT, handler)
signal.signal(signal.SIGTERM, handler)
return consumer
def create_batching_kafka_consumer(topic_names, worker, **options):
# In some cases we want to override the configuration stored in settings from the command line
force_topic = options.pop("force_topic", None)
force_cluster = options.pop("force_cluster", None)
if force_topic and force_cluster:
topic_names = {force_topic}
cluster_names = {force_cluster}
elif force_topic or force_cluster:
raise ValueError(
"Both 'force_topic' and 'force_cluster' have to be provided to override the configuration"
)
else:
cluster_names = {
settings.KAFKA_TOPICS[topic_name]["cluster"]
for topic_name in topic_names
}
if len(cluster_names) > 1:
raise ValueError(
f"Cannot launch Kafka consumer listening to multiple topics ({topic_names}) on different clusters ({cluster_names})"
)
(cluster_name, ) = cluster_names
consumer = BatchingKafkaConsumer(
topics=topic_names,
cluster_name=cluster_name,
worker=worker,
metrics=metrics,
metrics_default_tags={
"topics": ",".join(sorted(topic_names)),
"group_id": options.get("group_id"),
},
**options,
)
def handler(signum, frame):
consumer.signal_shutdown()
signal.signal(signal.SIGINT, handler)
signal.signal(signal.SIGTERM, handler)
return consumer
def _build_consumer(
self,
entity,
consumer_group,
commit_log_topic,
synchronize_commit_group,
commit_batch_size=100,
commit_batch_timeout_ms=5000,
initial_offset_reset="latest",
):
cluster_name = settings.KAFKA_TOPICS[settings.KAFKA_EVENTS]["cluster"]
synchronized_consumer = SynchronizedConsumer(
cluster_name=cluster_name,
consumer_group=consumer_group,
commit_log_topic=commit_log_topic,
synchronize_commit_group=synchronize_commit_group,
initial_offset_reset=initial_offset_reset,
)
concurrency = options.get(_CONCURRENCY_OPTION)
logger.info(
f"Starting post process forwrader to consume {entity} messages")
if entity == PostProcessForwarderType.TRANSACTIONS:
worker = TransactionsPostProcessForwarderWorker(
concurrency=concurrency)
elif entity == PostProcessForwarderType.ERRORS:
worker = ErrorsPostProcessForwarderWorker(concurrency=concurrency)
else:
# Default implementation which processes both errors and transactions
# irrespective of values in the header. This would most likely be the case
# for development environments.
worker = PostProcessForwarderWorker(concurrency=concurrency)
consumer = BatchingKafkaConsumer(
topics=self.topic,
worker=worker,
max_batch_size=commit_batch_size,
max_batch_time=commit_batch_timeout_ms,
consumer=synchronized_consumer,
commit_on_shutdown=True,
)
return consumer
def create_batching_kafka_consumer(topic_name, worker, **options):
cluster_name = settings.KAFKA_TOPICS[topic_name]["cluster"]
bootstrap_servers = settings.KAFKA_CLUSTERS[cluster_name][
"bootstrap.servers"]
if not isinstance(bootstrap_servers, (list, tuple)):
bootstrap_servers = bootstrap_servers.split(",")
consumer = BatchingKafkaConsumer(topics=[topic_name],
bootstrap_servers=bootstrap_servers,
worker=worker,
metrics=metrics,
metrics_default_tags={
"topic": topic_name,
"group_id": options.get("group_id")
},
**options)
def handler(signum, frame):
consumer.signal_shutdown()
signal.signal(signal.SIGINT, handler)
signal.signal(signal.SIGTERM, handler)
return consumer