help="Topic for committed offsets to be written to, triggering post-processing task(s)", ) @click.option( "--consumer-group", default="snuba-consumers", help="Consumer group use for consuming the raw events topic.", ) @click.option( "--bootstrap-server", multiple=True, help="Kafka bootstrap server to use.", ) @click.option( "--storage", "storage_name", type=click.Choice([storage_key.value for storage_key in WRITABLE_STORAGES.keys()]), help="The storage to target", required=True, ) @click.option( "--max-batch-size", default=settings.DEFAULT_MAX_BATCH_SIZE, type=int, help="Max number of messages to batch in memory before writing to Kafka.", ) @click.option( "--max-batch-time-ms", default=settings.DEFAULT_MAX_BATCH_TIME_MS, type=int, help="Max length of time to buffer messages in memory before writing to Kafka.", )
build_kafka_consumer_configuration, build_kafka_producer_configuration, ) from snuba.utils.streams.kafka_consumer_with_commit_log import ( KafkaConsumerWithCommitLog, ) from snuba.utils.streams.metrics_adapter import StreamMetricsAdapter logger = logging.getLogger(__name__) @click.command(hidden=True) @click.option( "--storage", "storage_names", type=click.Choice( [storage_key.value for storage_key in WRITABLE_STORAGES.keys()]), multiple=True, required=True, ) @click.option( "--consumer-group", default="snuba-consumers", ) @click.option( "--max-batch-size", default=settings.DEFAULT_MAX_BATCH_SIZE, type=int, help="Max number of messages to batch in memory before writing to Kafka.", ) @click.option( "--max-batch-time-ms",
@click.option( "--bootstrap-server", multiple=True, help="Kafka bootstrap server to use.", ) @click.option( "--dataset", "dataset_name", type=click.Choice(DATASET_NAMES), help="The dataset to target", ) @click.option( "--storage", "storage_name", default="events", type=click.Choice(WRITABLE_STORAGES.keys()), help="The storage to target", ) @click.option( "--max-batch-size", default=settings.DEFAULT_MAX_BATCH_SIZE, type=int, help="Max number of messages to batch in memory before writing to Kafka.", ) @click.option( "--max-batch-time-ms", default=settings.DEFAULT_MAX_BATCH_TIME_MS, type=int, help= "Max length of time to buffer messages in memory before writing to Kafka.", )