def test_post_process_forwarder_batch_consumer(
self, dispatch_post_process_group_task):
consumer_group = f"consumer-{uuid.uuid1().hex}"
synchronize_commit_group = f"sync-consumer-{uuid.uuid1().hex}"
events_producer = self._get_producer("events")
commit_log_producer = self._get_producer("snuba-commit-log")
message = json.dumps(kafka_message_payload()).encode()
eventstream = KafkaEventStream()
consumer = eventstream._build_consumer(
entity="all",
consumer_group=consumer_group,
commit_log_topic=self.commit_log_topic,
synchronize_commit_group=synchronize_commit_group,
commit_batch_size=1,
initial_offset_reset="earliest",
)
# produce message to the events topic
events_producer.produce(self.events_topic, message)
assert events_producer.flush(
5) == 0, "events producer did not successfully flush queue"
# Move the committed offset forward for our synchronizing group.
commit_log_producer.produce(
self.commit_log_topic,
key=f"{self.events_topic}:0:{synchronize_commit_group}".encode(),
value=f"{1}".encode(),
)
assert (commit_log_producer.flush(5) == 0
), "snuba-commit-log producer did not successfully flush queue"
# Run the loop for sometime
for _ in range(3):
consumer._run_once()
time.sleep(1)
# Verify that the task gets called once
dispatch_post_process_group_task.assert_called_once_with(
event_id="fe0ee9a2bc3b415497bad68aaf70dc7f",
project_id=1,
group_id=43,
primary_hash="311ee66a5b8e697929804ceb1c456ffe",
is_new=False,
is_regression=None,
is_new_group_environment=False,
)
def setUp(self):
super(SnubaEventStreamTest, self).setUp()
self.kafka_eventstream = KafkaEventStream()
self.kafka_eventstream.producer = Mock()
class SnubaEventStreamTest(TestCase, SnubaTestCase):
def setUp(self):
super(SnubaEventStreamTest, self).setUp()
self.kafka_eventstream = KafkaEventStream()
self.kafka_eventstream.producer = Mock()
def __build_event(self, timestamp):
raw_event = {
"event_id": "a" * 32,
"message": "foo",
"timestamp": time.mktime(timestamp.timetuple()),
"level": logging.ERROR,
"logger": "default",
"tags": [],
}
manager = EventManager(raw_event)
manager.normalize()
return manager.save(self.project.id)
def __produce_event(self, *insert_args, **insert_kwargs):
# pass arguments on to Kafka EventManager
self.kafka_eventstream.insert(*insert_args, **insert_kwargs)
produce_args, produce_kwargs = list(self.kafka_eventstream.producer.produce.call_args)
assert not produce_args
assert produce_kwargs["topic"] == "events"
assert produce_kwargs["key"] == six.text_type(self.project.id)
version, type_, payload1, payload2 = json.loads(produce_kwargs["value"])
assert version == 2
assert type_ == "insert"
# insert what would have been the Kafka payload directly
# into Snuba, expect an HTTP 200 and for the event to now exist
snuba_eventstream = SnubaEventStream()
snuba_eventstream._send(self.project.id, "insert", (payload1, payload2))
@patch("sentry.eventstream.insert")
@patch("sentry.tagstore.delay_index_event_tags")
def test(self, mock_delay_index_event_tags, mock_eventstream_insert):
now = datetime.utcnow()
def _get_event_count():
return snuba.query(
start=now - timedelta(days=1),
end=now + timedelta(days=1),
groupby=["project_id"],
filter_keys={"project_id": [self.project.id]},
).get(self.project.id, 0)
assert _get_event_count() == 0
event = self.__build_event(now)
# verify eventstream was called by EventManager
insert_args, insert_kwargs = list(mock_eventstream_insert.call_args)
assert not insert_args
assert insert_kwargs == {
"event": event,
"group": event.group,
"is_new_group_environment": True,
"is_new": True,
"is_regression": False,
"is_sample": False,
"primary_hash": "acbd18db4cc2f85cedef654fccc4a4d8",
"skip_consume": False,
}
assert mock_delay_index_event_tags.call_count == 1
self.__produce_event(*insert_args, **insert_kwargs)
assert _get_event_count() == 1
@patch("sentry.eventstream.insert")
@patch("sentry.tagstore.delay_index_event_tags")
def test_issueless(self, mock_delay_index_event_tags, mock_eventstream_insert):
now = datetime.utcnow()
event = self.__build_event(now)
event.group_id = None
insert_args = ()
insert_kwargs = {
"event": event,
"group": None,
"is_new_group_environment": True,
"is_new": True,
"is_regression": False,
"is_sample": False,
"primary_hash": "acbd18db4cc2f85cedef654fccc4a4d8",
"skip_consume": False,
}
self.__produce_event(*insert_args, **insert_kwargs)
result = snuba.raw_query(
start=now - timedelta(days=1),
end=now + timedelta(days=1),
selected_columns=["event_id", "group_id"],
groupby=None,
filter_keys={"project_id": [self.project.id], "event_id": [event.event_id]},
)
assert len(result["data"]) == 1
assert result["data"][0]["group_id"] is None
class SnubaEventStreamTest(SnubaTestCase):
def setUp(self):
super(SnubaEventStreamTest, self).setUp()
self.kafka_eventstream = KafkaEventStream()
self.kafka_eventstream.producer = Mock()
@patch('sentry.eventstream.insert')
@patch('sentry.tagstore.delay_index_event_tags')
def test(self, mock_delay_index_event_tags, mock_eventstream_insert):
now = datetime.utcnow()
def _get_event_count():
return snuba.query(
start=now - timedelta(days=1),
end=now + timedelta(days=1),
groupby=['project_id'],
filter_keys={'project_id': [self.project.id]},
).get(self.project.id, 0)
assert _get_event_count() == 0
raw_event = {
'event_id': 'a' * 32,
'message': 'foo',
'timestamp': time.mktime(now.timetuple()),
'level': logging.ERROR,
'logger': 'default',
'tags': [],
}
manager = EventManager(raw_event)
manager.normalize()
event = manager.save(self.project.id)
# verify eventstream was called by EventManager
insert_args, insert_kwargs = list(mock_eventstream_insert.call_args)
assert not insert_args
assert insert_kwargs == {
'event': event,
'group': event.group,
'is_new_group_environment': True,
'is_new': True,
'is_regression': False,
'is_sample': False,
'primary_hash': 'acbd18db4cc2f85cedef654fccc4a4d8',
'skip_consume': False
}
assert mock_delay_index_event_tags.call_count == 1
# pass arguments on to Kafka EventManager
self.kafka_eventstream.insert(*insert_args, **insert_kwargs)
produce_args, produce_kwargs = list(self.kafka_eventstream.producer.produce.call_args)
assert not produce_args
assert produce_kwargs['topic'] == 'events'
assert produce_kwargs['key'] == six.text_type(self.project.id)
version, type_, payload1, payload2 = json.loads(produce_kwargs['value'])
assert version == 2
assert type_ == 'insert'
# insert what would have been the Kafka payload directly
# into Snuba, expect an HTTP 200 and for the event to now exist
snuba_eventstream = SnubaEventStream()
snuba_eventstream._send(self.project.id, 'insert', (payload1, payload2))
assert _get_event_count() == 1
def setUp(self):
super(SnubaEventStreamTest, self).setUp()
self.kafka_eventstream = KafkaEventStream()
self.kafka_eventstream.producer = Mock()
class SnubaEventStreamTest(TestCase, SnubaTestCase):
def setUp(self):
super(SnubaEventStreamTest, self).setUp()
self.kafka_eventstream = KafkaEventStream()
self.kafka_eventstream.producer = Mock()
@patch('sentry.eventstream.insert')
@patch('sentry.tagstore.delay_index_event_tags')
def test(self, mock_delay_index_event_tags, mock_eventstream_insert):
now = datetime.utcnow()
def _get_event_count():
return snuba.query(
start=now - timedelta(days=1),
end=now + timedelta(days=1),
groupby=['project_id'],
filter_keys={'project_id': [self.project.id]},
).get(self.project.id, 0)
assert _get_event_count() == 0
raw_event = {
'event_id': 'a' * 32,
'message': 'foo',
'timestamp': time.mktime(now.timetuple()),
'level': logging.ERROR,
'logger': 'default',
'tags': [],
}
manager = EventManager(raw_event)
manager.normalize()
event = manager.save(self.project.id)
# verify eventstream was called by EventManager
insert_args, insert_kwargs = list(mock_eventstream_insert.call_args)
assert not insert_args
assert insert_kwargs == {
'event': event,
'group': event.group,
'is_new_group_environment': True,
'is_new': True,
'is_regression': False,
'is_sample': False,
'primary_hash': 'acbd18db4cc2f85cedef654fccc4a4d8',
'skip_consume': False
}
assert mock_delay_index_event_tags.call_count == 1
# pass arguments on to Kafka EventManager
self.kafka_eventstream.insert(*insert_args, **insert_kwargs)
produce_args, produce_kwargs = list(self.kafka_eventstream.producer.produce.call_args)
assert not produce_args
assert produce_kwargs['topic'] == 'events'
assert produce_kwargs['key'] == six.text_type(self.project.id)
version, type_, payload1, payload2 = json.loads(produce_kwargs['value'])
assert version == 2
assert type_ == 'insert'
# insert what would have been the Kafka payload directly
# into Snuba, expect an HTTP 200 and for the event to now exist
snuba_eventstream = SnubaEventStream()
snuba_eventstream._send(self.project.id, 'insert', (payload1, payload2))
assert _get_event_count() == 1
