def test_one_message(self, create_consumer) -> None: kafka_consumer = FakeConfluentKafkaConsumer() create_consumer.return_value = kafka_consumer msg = build_confluent_kafka_message(0, 0, b"ABCABC", False) kafka_consumer.items = [ msg, build_confluent_kafka_message(0, 0, None, True), ] on_message = MagicMock() on_message.return_value = CommitDecision.DO_NOT_COMMIT consumer = self.__consumer(on_message) consumer.run() on_message.assert_called_once_with(msg) assert kafka_consumer.commit_calls == 0
def test_commits(self, create_consumer) -> None: kafka_consumer = FakeConfluentKafkaConsumer() create_consumer.return_value = kafka_consumer error = MagicMock() error.code.return_value = KafkaError._PARTITION_EOF kafka_consumer.items = [ build_confluent_kafka_message(0, 0, b"ABCABC", False), build_confluent_kafka_message(1, 0, b"ABCABC", False), build_confluent_kafka_message(2, 0, b"ABCABC", False), build_confluent_kafka_message(0, 0, None, True), ] on_message = MagicMock() on_message.return_value = CommitDecision.COMMIT_PREV consumer = self.__consumer(on_message) consumer.run() on_message.assert_called() assert kafka_consumer.commit_calls == 2
def test_empty_topic(self, create_consumer) -> None: kafka_consumer = FakeConfluentKafkaConsumer() kafka_consumer.items = [ build_confluent_kafka_message(0, 0, None, True), ] create_consumer.return_value = kafka_consumer on_message = MagicMock() consumer = self.__consumer(on_message) consumer.run() on_message.assert_not_called()
def test_snapshot_loaded(self, create_consumer) -> None: kafka_consumer = FakeConfluentKafkaConsumer() kafka_consumer.items = [ build_confluent_kafka_message( 0, 0, b'{"snapshot-id":"abc123", "product":"somewhere-else", "tables": [], "event":"snapshot-init"}', False, ), build_confluent_kafka_message( 1, 0, b'{"snapshot-id":"abc123", "product":"snuba", "tables": ["sentry_groupedmessage"], "event":"snapshot-init"}', False, ), build_confluent_kafka_message( 2, 0, ( b'{"snapshot-id":"abc123", "event":"snapshot-loaded",' b'"transaction-info": {"xmin":123, "xmax":124, "xip-list": []}' b'}' ), False, ), build_confluent_kafka_message(0, 0, None, True), ] create_consumer.return_value = kafka_consumer bootstrap = BootstrapState( "cdc_control", "somewhere", "something", get_dataset("groupedmessage"), ) ret = bootstrap.handle(None) assert ret[0] == ConsumerStateCompletionEvent.SNAPSHOT_READY_RECEIVED assert kafka_consumer.commit_calls == 2
def test_init_snapshot(self, create_consumer) -> None: kafka_consumer = FakeConfluentKafkaConsumer() kafka_consumer.items = [ build_confluent_kafka_message( 0, 0, b'{"snapshot-id":"abc123", "tables": ["sentry_groupedmessage"], "product":"snuba", "event":"snapshot-init"}', False, ), build_confluent_kafka_message(0, 0, None, True), ] create_consumer.return_value = kafka_consumer bootstrap = BootstrapState( "cdc_control", "somewhere", "something", get_dataset("groupedmessage"), ) ret = bootstrap.handle(None) assert ret[0] == ConsumerStateCompletionEvent.SNAPSHOT_INIT_RECEIVED assert kafka_consumer.commit_calls == 0
def test_snapshot_for_other_table(self, create_consumer) -> None: kafka_consumer = FakeConfluentKafkaConsumer() kafka_consumer.items = [ build_confluent_kafka_message( 0, 0, b'{"snapshot-id":"abc123", "tables": ["someone_else"], "product":"snuba", "event":"snapshot-init"}', False, ), build_confluent_kafka_message(0, 0, None, True), ] create_consumer.return_value = kafka_consumer bootstrap = BootstrapState( "cdc_control", self.broker_config, "something", get_cdc_storage(StorageKey.GROUPEDMESSAGES), ) ret = bootstrap.handle(None) assert ret[0] == ConsumerStateCompletionEvent.NO_SNAPSHOT assert kafka_consumer.commit_calls == 1
def test_empty_topic(self, create_consumer) -> None: kafka_consumer = FakeConfluentKafkaConsumer() kafka_consumer.items = [ build_confluent_kafka_message(0, 0, None, True), ] create_consumer.return_value = kafka_consumer bootstrap = BootstrapState( "cdc_control", "somewhere", "something", get_dataset("groupedmessage"), ) ret = bootstrap.handle(None) assert ret[0] == ConsumerStateCompletionEvent.NO_SNAPSHOT assert kafka_consumer.commit_calls == 0
def test_empty_topic(self, create_consumer) -> None: kafka_consumer = FakeConfluentKafkaConsumer() kafka_consumer.items = [ build_confluent_kafka_message(0, 0, None, True), ] create_consumer.return_value = kafka_consumer bootstrap = BootstrapState( "cdc_control", self.broker_config, "something", get_cdc_storage(StorageKey.GROUPEDMESSAGES), ) ret = bootstrap.handle(None) assert ret[0] == ConsumerStateCompletionEvent.NO_SNAPSHOT assert kafka_consumer.commit_calls == 0