def test_one_message(self, create_consumer) -> None:
kafka_consumer = FakeConfluentKafkaConsumer()
create_consumer.return_value = kafka_consumer
msg = build_confluent_kafka_message(0, 0, b"ABCABC", False)
kafka_consumer.items = [
msg,
build_confluent_kafka_message(0, 0, None, True),
]
on_message = MagicMock()
on_message.return_value = CommitDecision.DO_NOT_COMMIT
consumer = self.__consumer(on_message)
consumer.run()
on_message.assert_called_once_with(msg)
assert kafka_consumer.commit_calls == 0
def test_commits(self, create_consumer) -> None:
kafka_consumer = FakeConfluentKafkaConsumer()
create_consumer.return_value = kafka_consumer
error = MagicMock()
error.code.return_value = KafkaError._PARTITION_EOF
kafka_consumer.items = [
build_confluent_kafka_message(0, 0, b"ABCABC", False),
build_confluent_kafka_message(1, 0, b"ABCABC", False),
build_confluent_kafka_message(2, 0, b"ABCABC", False),
build_confluent_kafka_message(0, 0, None, True),
]
on_message = MagicMock()
on_message.return_value = CommitDecision.COMMIT_PREV
consumer = self.__consumer(on_message)
consumer.run()
on_message.assert_called()
assert kafka_consumer.commit_calls == 2
def test_empty_topic(self, create_consumer) -> None:
kafka_consumer = FakeConfluentKafkaConsumer()
kafka_consumer.items = [
build_confluent_kafka_message(0, 0, None, True),
]
create_consumer.return_value = kafka_consumer
on_message = MagicMock()
consumer = self.__consumer(on_message)
consumer.run()
on_message.assert_not_called()
def test_snapshot_loaded(self, create_consumer) -> None:
kafka_consumer = FakeConfluentKafkaConsumer()
kafka_consumer.items = [
build_confluent_kafka_message(
0,
0,
b'{"snapshot-id":"abc123", "product":"somewhere-else", "tables": [], "event":"snapshot-init"}',
False,
),
build_confluent_kafka_message(
1,
0,
b'{"snapshot-id":"abc123", "product":"snuba", "tables": ["sentry_groupedmessage"], "event":"snapshot-init"}',
False,
),
build_confluent_kafka_message(
2,
0,
(
b'{"snapshot-id":"abc123", "event":"snapshot-loaded",'
b'"transaction-info": {"xmin":123, "xmax":124, "xip-list": []}'
b'}'
),
False,
),
build_confluent_kafka_message(0, 0, None, True),
]
create_consumer.return_value = kafka_consumer
bootstrap = BootstrapState(
"cdc_control",
"somewhere",
"something",
get_dataset("groupedmessage"),
)
ret = bootstrap.handle(None)
assert ret[0] == ConsumerStateCompletionEvent.SNAPSHOT_READY_RECEIVED
assert kafka_consumer.commit_calls == 2
def test_init_snapshot(self, create_consumer) -> None:
kafka_consumer = FakeConfluentKafkaConsumer()
kafka_consumer.items = [
build_confluent_kafka_message(
0,
0,
b'{"snapshot-id":"abc123", "tables": ["sentry_groupedmessage"], "product":"snuba", "event":"snapshot-init"}',
False,
),
build_confluent_kafka_message(0, 0, None, True),
]
create_consumer.return_value = kafka_consumer
bootstrap = BootstrapState(
"cdc_control",
"somewhere",
"something",
get_dataset("groupedmessage"),
)
ret = bootstrap.handle(None)
assert ret[0] == ConsumerStateCompletionEvent.SNAPSHOT_INIT_RECEIVED
assert kafka_consumer.commit_calls == 0
def test_snapshot_for_other_table(self, create_consumer) -> None:
kafka_consumer = FakeConfluentKafkaConsumer()
kafka_consumer.items = [
build_confluent_kafka_message(
0,
0,
b'{"snapshot-id":"abc123", "tables": ["someone_else"], "product":"snuba", "event":"snapshot-init"}',
False,
),
build_confluent_kafka_message(0, 0, None, True),
]
create_consumer.return_value = kafka_consumer
bootstrap = BootstrapState(
"cdc_control",
self.broker_config,
"something",
get_cdc_storage(StorageKey.GROUPEDMESSAGES),
)
ret = bootstrap.handle(None)
assert ret[0] == ConsumerStateCompletionEvent.NO_SNAPSHOT
assert kafka_consumer.commit_calls == 1
def test_empty_topic(self, create_consumer) -> None:
kafka_consumer = FakeConfluentKafkaConsumer()
kafka_consumer.items = [
build_confluent_kafka_message(0, 0, None, True),
]
create_consumer.return_value = kafka_consumer
bootstrap = BootstrapState(
"cdc_control",
"somewhere",
"something",
get_dataset("groupedmessage"),
)
ret = bootstrap.handle(None)
assert ret[0] == ConsumerStateCompletionEvent.NO_SNAPSHOT
assert kafka_consumer.commit_calls == 0
def test_empty_topic(self, create_consumer) -> None:
kafka_consumer = FakeConfluentKafkaConsumer()
kafka_consumer.items = [
build_confluent_kafka_message(0, 0, None, True),
]
create_consumer.return_value = kafka_consumer
bootstrap = BootstrapState(
"cdc_control",
self.broker_config,
"something",
get_cdc_storage(StorageKey.GROUPEDMESSAGES),
)
ret = bootstrap.handle(None)
assert ret[0] == ConsumerStateCompletionEvent.NO_SNAPSHOT
assert kafka_consumer.commit_calls == 0
