def ingest_consumer(consumer_types, all_consumer_types, **options):
"""
Runs an "ingest consumer" task.
The "ingest consumer" tasks read events from a kafka topic (coming from Relay) and schedules
process event celery tasks for them
"""
from sentry.ingest.ingest_consumer import get_ingest_consumer
from sentry.utils import metrics
if all_consumer_types:
if consumer_types:
raise click.ClickException(
"Cannot specify --all-consumer types and --consumer-type at the same time"
)
else:
consumer_types = set(ConsumerType.all())
if not all_consumer_types and not consumer_types:
raise click.ClickException(
"Need to specify --all-consumer-types or --consumer-type")
concurrency = options.pop("concurrency", None)
if concurrency is not None:
executor = ThreadPoolExecutor(concurrency)
else:
executor = None
with metrics.global_tags(ingest_consumer_types=",".join(
sorted(consumer_types)),
_all_threads=True):
get_ingest_consumer(consumer_types=consumer_types,
executor=executor,
**options).run()
def ingest_consumer(settings):
from sentry.ingest.ingest_consumer import ConsumerType, get_ingest_consumer
if scope_consumers["ingest_events"] is not None:
return scope_consumers[
"ingest_events"] # reuse whatever was already created (will ignore the settings)
# first time the consumer is requested, create it using settings
topic_event_name = ConsumerType.get_topic_name(ConsumerType.Events)
admin = kafka_admin(settings)
admin.delete_topic(topic_event_name)
# simulate the event ingestion task
group_id = "test-consumer"
consumer = get_ingest_consumer(
max_batch_size=1,
max_batch_time=10,
group_id=group_id,
consumer_types=[ConsumerType.Events],
auto_offset_reset="earliest",
)
scope_consumers["ingest_events"] = consumer
return consumer
def ingest_consumer(consumer_type, **options):
"""
Runs an "ingest consumer" task.
The "ingest consumer" tasks read events from a kafka topic (coming from Relay) and schedules
process event celery tasks for them
"""
from sentry.ingest.ingest_consumer import ConsumerType, get_ingest_consumer
if consumer_type == "events":
consumer_type = ConsumerType.Events
elif consumer_type == "transactions":
consumer_type = ConsumerType.Transactions
elif consumer_type == "attachments":
consumer_type = ConsumerType.Attachments
get_ingest_consumer(consumer_type=consumer_type, **options).run()
def ingest_consumer(consumer_type, **options):
"""
Runs an "ingest consumer" task.
The "ingest consumer" tasks read events from a kafka topic (coming from Relay) and schedules
process event celery tasks for them
"""
from sentry.ingest.ingest_consumer import ConsumerType, get_ingest_consumer
from sentry.utils import metrics
if consumer_type == "events":
consumer_type = ConsumerType.Events
elif consumer_type == "transactions":
consumer_type = ConsumerType.Transactions
elif consumer_type == "attachments":
consumer_type = ConsumerType.Attachments
with metrics.global_tags(ingest_consumer_type=consumer_type,
_all_threads=True):
get_ingest_consumer(consumer_type=consumer_type, **options).run()
def test_ingest_topic_can_be_overridden(
task_runner,
kafka_admin,
requires_kafka,
random_group_id,
default_project,
get_test_message,
kafka_producer,
):
"""
Tests that 'force_topic' overrides the value provided in settings
"""
default_event_topic = ConsumerType.get_topic_name(ConsumerType.Events)
new_event_topic = default_event_topic + "-new"
admin = kafka_admin(settings)
admin.delete_topic(default_event_topic)
admin.delete_topic(new_event_topic)
producer = kafka_producer(settings)
message, event_id = get_test_message(type="event")
producer.produce(new_event_topic, message)
with override_settings(KAFKA_CONSUMER_AUTO_CREATE_TOPICS=True):
consumer = get_ingest_consumer(
max_batch_size=2,
max_batch_time=5000,
group_id=random_group_id,
consumer_types={ConsumerType.Events},
auto_offset_reset="earliest",
force_topic=new_event_topic,
force_cluster="default",
)
with task_runner():
i = 0
while i < MAX_POLL_ITERATIONS:
message = eventstore.get_event_by_id(default_project.id, event_id)
if message:
break
consumer._run_once()
i += 1
# Check that we got the message
assert message.data["event_id"] == event_id
assert message.data["extra"]["the_id"] == event_id
# Check that the default topic was not created
all_topics = admin.admin_client.list_topics().topics.keys()
assert new_event_topic in all_topics
assert default_event_topic not in all_topics
def test_ingest_consumer_reads_from_topic_and_calls_celery_task(
executor,
task_runner,
kafka_producer,
kafka_admin,
requires_kafka,
default_project,
get_test_message,
random_group_id,
):
topic_event_name = ConsumerType.get_topic_name(ConsumerType.Events)
admin = kafka_admin(settings)
admin.delete_topic(topic_event_name)
producer = kafka_producer(settings)
message, event_id = get_test_message(type="event")
producer.produce(topic_event_name, message)
transaction_message, transaction_event_id = get_test_message(type="transaction")
producer.produce(topic_event_name, transaction_message)
with override_settings(KAFKA_CONSUMER_AUTO_CREATE_TOPICS=True):
consumer = get_ingest_consumer(
max_batch_size=2,
max_batch_time=5000,
group_id=random_group_id,
consumer_types={ConsumerType.Events},
auto_offset_reset="earliest",
)
with task_runner():
i = 0
while i < MAX_POLL_ITERATIONS:
transaction_message = eventstore.get_event_by_id(
default_project.id, transaction_event_id
)
message = eventstore.get_event_by_id(default_project.id, event_id)
if transaction_message and message:
break
consumer._run_once()
i += 1
# check that we got the messages
assert message.data["event_id"] == event_id
assert message.data["extra"]["the_id"] == event_id
assert transaction_message.data["event_id"] == transaction_event_id
assert transaction_message.data["spans"] == []
assert transaction_message.data["contexts"]["trace"]
def test_ingest_consumer_reads_from_topic_and_calls_celery_task(
task_runner,
kafka_producer,
kafka_admin,
requires_kafka,
default_project,
get_test_message,
inline_transactions,
):
group_id = "test-consumer"
topic_event_name = ConsumerType.get_topic_name(ConsumerType.Events)
admin = kafka_admin(settings)
admin.delete_topic(topic_event_name)
producer = kafka_producer(settings)
event_ids = set()
for _ in range(3):
message, event_id = get_test_message()
producer.produce(topic_event_name, message)
event_ids.add(event_id)
consumer = get_ingest_consumer(
max_batch_size=2,
max_batch_time=5000,
group_id=group_id,
consumer_types=set([ConsumerType.Events]),
auto_offset_reset="earliest",
)
options.set("store.transactions-celery", not inline_transactions)
with task_runner():
i = 0
while i < MAX_POLL_ITERATIONS:
if eventstore.get_event_by_id(default_project.id, event_id):
break
consumer._run_once()
i += 1
# check that we got the messages
for event_id in event_ids:
message = eventstore.get_event_by_id(default_project.id, event_id)
assert message is not None
assert message.data["event_id"] == event_id
if message.data["type"] == "transaction":
assert message.data["spans"] == []
assert message.data["contexts"]["trace"]
else:
assert message.data["extra"]["the_id"] == event_id
def test_ingest_consumer_reads_from_topic_and_calls_celery_task(
task_runner, kafka_producer, kafka_admin, requires_kafka):
group_id = "test-consumer"
topic_event_name = ConsumerType.get_topic_name(ConsumerType.Events)
admin = kafka_admin(settings)
admin.delete_topic(topic_event_name)
producer = kafka_producer(settings)
organization = Factories.create_organization()
project = Factories.create_project(organization=organization)
event_ids = set()
for _ in range(3):
message, event_id = _get_test_message(project)
event_ids.add(event_id)
producer.produce(topic_event_name, message)
consumer = get_ingest_consumer(
max_batch_size=2,
max_batch_time=5000,
group_id=group_id,
consumer_type=ConsumerType.Events,
auto_offset_reset="earliest",
)
with task_runner():
i = 0
while i < MAX_POLL_ITERATIONS:
if eventstore.get_event_by_id(project.id, event_id):
break
consumer._run_once()
i += 1
# check that we got the messages
for event_id in event_ids:
message = eventstore.get_event_by_id(project.id, event_id)
assert message is not None
# check that the data has not been scrambled
assert message.data["extra"]["the_id"] == event_id
def test_ingest_consumer_fails_when_not_autocreating_topics(
kafka_admin, requires_kafka, random_group_id):
topic_event_name = ConsumerType.get_topic_name(ConsumerType.Events)
admin = kafka_admin(settings)
admin.delete_topic(topic_event_name)
with override_settings(KAFKA_CONSUMER_AUTO_CREATE_TOPICS=False):
consumer = get_ingest_consumer(
max_batch_size=2,
max_batch_time=5000,
group_id=random_group_id,
consumer_types={ConsumerType.Events},
auto_offset_reset="earliest",
)
with pytest.raises(Exception) as err:
consumer._run_once()
kafka_error = err.value.args[0]
assert kafka_error.code() == KafkaError.UNKNOWN_TOPIC_OR_PART
