help="Topic for committed offsets to be written to, triggering post-processing task(s)",
)
@click.option(
"--consumer-group",
default="snuba-consumers",
help="Consumer group use for consuming the raw events topic.",
)
@click.option(
"--bootstrap-server",
multiple=True,
help="Kafka bootstrap server to use.",
)
@click.option(
"--storage",
"storage_name",
type=click.Choice([storage_key.value for storage_key in WRITABLE_STORAGES.keys()]),
help="The storage to target",
required=True,
)
@click.option(
"--max-batch-size",
default=settings.DEFAULT_MAX_BATCH_SIZE,
type=int,
help="Max number of messages to batch in memory before writing to Kafka.",
)
@click.option(
"--max-batch-time-ms",
default=settings.DEFAULT_MAX_BATCH_TIME_MS,
type=int,
help="Max length of time to buffer messages in memory before writing to Kafka.",
)
build_kafka_consumer_configuration,
build_kafka_producer_configuration,
)
from snuba.utils.streams.kafka_consumer_with_commit_log import (
KafkaConsumerWithCommitLog, )
from snuba.utils.streams.metrics_adapter import StreamMetricsAdapter
logger = logging.getLogger(__name__)
@click.command(hidden=True)
@click.option(
"--storage",
"storage_names",
type=click.Choice(
[storage_key.value for storage_key in WRITABLE_STORAGES.keys()]),
multiple=True,
required=True,
)
@click.option(
"--consumer-group",
default="snuba-consumers",
)
@click.option(
"--max-batch-size",
default=settings.DEFAULT_MAX_BATCH_SIZE,
type=int,
help="Max number of messages to batch in memory before writing to Kafka.",
)
@click.option(
"--max-batch-time-ms",
@click.option(
"--bootstrap-server",
multiple=True,
help="Kafka bootstrap server to use.",
)
@click.option(
"--dataset",
"dataset_name",
type=click.Choice(DATASET_NAMES),
help="The dataset to target",
)
@click.option(
"--storage",
"storage_name",
default="events",
type=click.Choice(WRITABLE_STORAGES.keys()),
help="The storage to target",
)
@click.option(
"--max-batch-size",
default=settings.DEFAULT_MAX_BATCH_SIZE,
type=int,
help="Max number of messages to batch in memory before writing to Kafka.",
)
@click.option(
"--max-batch-time-ms",
default=settings.DEFAULT_MAX_BATCH_TIME_MS,
type=int,
help=
"Max length of time to buffer messages in memory before writing to Kafka.",
)