def ingest_consumer(settings): from sentry.ingest.ingest_consumer import ConsumerType, get_ingest_consumer if scope_consumers["ingest_events"] is not None: return scope_consumers[ "ingest_events"] # reuse whatever was already created (will ignore the settings) # first time the consumer is requested, create it using settings topic_event_name = ConsumerType.get_topic_name(ConsumerType.Events) admin = kafka_admin(settings) admin.delete_topic(topic_event_name) # simulate the event ingestion task group_id = "test-consumer" consumer = get_ingest_consumer( max_batch_size=1, max_batch_time=10, group_id=group_id, consumer_types=[ConsumerType.Events], auto_offset_reset="earliest", ) scope_consumers["ingest_events"] = consumer return consumer
def test_ingest_topic_can_be_overridden( task_runner, kafka_admin, requires_kafka, random_group_id, default_project, get_test_message, kafka_producer, ): """ Tests that 'force_topic' overrides the value provided in settings """ default_event_topic = ConsumerType.get_topic_name(ConsumerType.Events) new_event_topic = default_event_topic + "-new" admin = kafka_admin(settings) admin.delete_topic(default_event_topic) admin.delete_topic(new_event_topic) producer = kafka_producer(settings) message, event_id = get_test_message(type="event") producer.produce(new_event_topic, message) with override_settings(KAFKA_CONSUMER_AUTO_CREATE_TOPICS=True): consumer = get_ingest_consumer( max_batch_size=2, max_batch_time=5000, group_id=random_group_id, consumer_types={ConsumerType.Events}, auto_offset_reset="earliest", force_topic=new_event_topic, force_cluster="default", ) with task_runner(): i = 0 while i < MAX_POLL_ITERATIONS: message = eventstore.get_event_by_id(default_project.id, event_id) if message: break consumer._run_once() i += 1 # Check that we got the message assert message.data["event_id"] == event_id assert message.data["extra"]["the_id"] == event_id # Check that the default topic was not created all_topics = admin.admin_client.list_topics().topics.keys() assert new_event_topic in all_topics assert default_event_topic not in all_topics
def test_ingest_consumer_reads_from_topic_and_calls_celery_task( executor, task_runner, kafka_producer, kafka_admin, requires_kafka, default_project, get_test_message, random_group_id, ): topic_event_name = ConsumerType.get_topic_name(ConsumerType.Events) admin = kafka_admin(settings) admin.delete_topic(topic_event_name) producer = kafka_producer(settings) message, event_id = get_test_message(type="event") producer.produce(topic_event_name, message) transaction_message, transaction_event_id = get_test_message(type="transaction") producer.produce(topic_event_name, transaction_message) with override_settings(KAFKA_CONSUMER_AUTO_CREATE_TOPICS=True): consumer = get_ingest_consumer( max_batch_size=2, max_batch_time=5000, group_id=random_group_id, consumer_types={ConsumerType.Events}, auto_offset_reset="earliest", ) with task_runner(): i = 0 while i < MAX_POLL_ITERATIONS: transaction_message = eventstore.get_event_by_id( default_project.id, transaction_event_id ) message = eventstore.get_event_by_id(default_project.id, event_id) if transaction_message and message: break consumer._run_once() i += 1 # check that we got the messages assert message.data["event_id"] == event_id assert message.data["extra"]["the_id"] == event_id assert transaction_message.data["event_id"] == transaction_event_id assert transaction_message.data["spans"] == [] assert transaction_message.data["contexts"]["trace"]
def test_ingest_consumer_reads_from_topic_and_calls_celery_task( task_runner, kafka_producer, kafka_admin, requires_kafka, default_project, get_test_message, inline_transactions, ): group_id = "test-consumer" topic_event_name = ConsumerType.get_topic_name(ConsumerType.Events) admin = kafka_admin(settings) admin.delete_topic(topic_event_name) producer = kafka_producer(settings) event_ids = set() for _ in range(3): message, event_id = get_test_message() producer.produce(topic_event_name, message) event_ids.add(event_id) consumer = get_ingest_consumer( max_batch_size=2, max_batch_time=5000, group_id=group_id, consumer_types=set([ConsumerType.Events]), auto_offset_reset="earliest", ) options.set("store.transactions-celery", not inline_transactions) with task_runner(): i = 0 while i < MAX_POLL_ITERATIONS: if eventstore.get_event_by_id(default_project.id, event_id): break consumer._run_once() i += 1 # check that we got the messages for event_id in event_ids: message = eventstore.get_event_by_id(default_project.id, event_id) assert message is not None assert message.data["event_id"] == event_id if message.data["type"] == "transaction": assert message.data["spans"] == [] assert message.data["contexts"]["trace"] else: assert message.data["extra"]["the_id"] == event_id
def test_ingest_consumer_reads_from_topic_and_calls_celery_task( task_runner, kafka_producer, kafka_admin, requires_kafka): group_id = "test-consumer" topic_event_name = ConsumerType.get_topic_name(ConsumerType.Events) admin = kafka_admin(settings) admin.delete_topic(topic_event_name) producer = kafka_producer(settings) organization = Factories.create_organization() project = Factories.create_project(organization=organization) event_ids = set() for _ in range(3): message, event_id = _get_test_message(project) event_ids.add(event_id) producer.produce(topic_event_name, message) consumer = get_ingest_consumer( max_batch_size=2, max_batch_time=5000, group_id=group_id, consumer_type=ConsumerType.Events, auto_offset_reset="earliest", ) with task_runner(): i = 0 while i < MAX_POLL_ITERATIONS: if eventstore.get_event_by_id(project.id, event_id): break consumer._run_once() i += 1 # check that we got the messages for event_id in event_ids: message = eventstore.get_event_by_id(project.id, event_id) assert message is not None # check that the data has not been scrambled assert message.data["extra"]["the_id"] == event_id
def test_ingest_consumer_fails_when_not_autocreating_topics( kafka_admin, requires_kafka, random_group_id): topic_event_name = ConsumerType.get_topic_name(ConsumerType.Events) admin = kafka_admin(settings) admin.delete_topic(topic_event_name) with override_settings(KAFKA_CONSUMER_AUTO_CREATE_TOPICS=False): consumer = get_ingest_consumer( max_batch_size=2, max_batch_time=5000, group_id=random_group_id, consumer_types={ConsumerType.Events}, auto_offset_reset="earliest", ) with pytest.raises(Exception) as err: consumer._run_once() kafka_error = err.value.args[0] assert kafka_error.code() == KafkaError.UNKNOWN_TOPIC_OR_PART
def test_ingest_consumer_reads_from_topic_and_calls_celery_task( task_runner, kafka_producer, kafka_admin): consumer_group = "test-consumer" topic_event_name = ConsumerType.get_topic_name(ConsumerType.Events) admin = kafka_admin(settings) admin.delete_topic(topic_event_name) producer = kafka_producer(settings) organization = Factories.create_organization() project = Factories.create_project(organization=organization) event_ids = set() for _ in range(3): message, event_id = _get_test_message(project) event_ids.add(event_id) producer.produce(topic_event_name, message) with task_runner(): run_ingest_consumer( commit_batch_size=2, consumer_group=consumer_group, consumer_type=ConsumerType.Events, max_fetch_time_seconds=0.1, initial_offset_reset="earliest", is_shutdown_requested=_shutdown_requested(max_secs=10, num_events=3), ) # check that we got the messages assert Event.objects.count() == 3 for event_id in event_ids: message = Event.objects.get(event_id=event_id) assert message is not None # check that the data has not been scrambled assert message.data["extra"]["the_id"] == event_id