def ingest_consumer(consumer_types, all_consumer_types, **options):
"""
Runs an "ingest consumer" task.
The "ingest consumer" tasks read events from a kafka topic (coming from Relay) and schedules
process event celery tasks for them
"""
from sentry.ingest.ingest_consumer import get_ingest_consumer
from sentry.utils import metrics
if all_consumer_types:
if consumer_types:
raise click.ClickException(
"Cannot specify --all-consumer types and --consumer-type at the same time"
)
else:
consumer_types = set(ConsumerType.all())
if not all_consumer_types and not consumer_types:
raise click.ClickException(
"Need to specify --all-consumer-types or --consumer-type")
concurrency = options.pop("concurrency", None)
if concurrency is not None:
executor = ThreadPoolExecutor(concurrency)
else:
executor = None
with metrics.global_tags(ingest_consumer_types=",".join(
sorted(consumer_types)),
_all_threads=True):
get_ingest_consumer(consumer_types=consumer_types,
executor=executor,
**options).run()
def get_ingest_consumer(consumer_types, once=False, **options):
"""
Handles events coming via a kafka queue.
The events should have already been processed (normalized... ) upstream (by Relay).
"""
topic_names = {ConsumerType.get_topic_name(consumer_type) for consumer_type in consumer_types}
return create_batching_kafka_consumer(
topic_names=topic_names, worker=IngestConsumerWorker(), **options
)
return f
return inner
@run.command("ingest-consumer")
@log_options()
@click.option(
"consumer_types",
"--consumer-type",
default=[],
multiple=True,
help=
"Specify which type of consumer to create, i.e. from which topic to consume messages. By default all ingest-related topics are consumed ",
type=click.Choice(ConsumerType.all()),
)
@click.option(
"--all-consumer-types",
default=False,
is_flag=True,
help="Listen to all consumer types at once.",
)
@batching_kafka_options("ingest-consumer")
@click.option(
"--concurrency",
type=int,
default=None,
help=
"Thread pool size (only utilitized for message types that support concurrent processing)",
)