def test_set_offsets_offset_to_delta_all_topics(
topic: str,
interactive_cli_runner,
producer: ConfluenceProducer,
consumer_group: str,
consumergroup_controller: ConsumerGroupController,
):
produce_text_test_messages(producer=producer, topic_name=topic, amount=10)
consumergroup_controller.commit_offsets(
consumer_group, [TopicPartition(topic=topic, partition=0, offset=10)])
consumergroup_desc_before = consumergroup_controller.get_consumer_group(
consumer_id=consumer_group).describe(partitions=True)
interactive_cli_runner.invoke(
esque,
args=["set", "offsets", consumer_group, "--offset-by-delta", "-2"],
input="y\n",
catch_exceptions=False)
# Check assertions:
consumergroup_desc_after = consumergroup_controller.get_consumer_group(
consumer_id=consumer_group).describe(partitions=True)
assert consumergroup_desc_before["offsets"][topic][0][
"consumer_offset"] == 10
assert consumergroup_desc_after["offsets"][topic][0][
"consumer_offset"] == 8
def test_edit_offsets(
monkeypatch: MonkeyPatch,
interactive_cli_runner,
topic: str,
producer: ConfluenceProducer,
consumer_group: str,
consumergroup_controller: ConsumerGroupController,
):
produce_text_test_messages(producer=producer, topic_name=topic, amount=10)
consumergroup_controller.commit_offsets(consumer_group, [TopicPartition(topic=topic, partition=0, offset=10)])
consumergroup_desc_before = consumergroup_controller.get_consumer_group(consumer_id=consumer_group).describe(
partitions=True
)
offset_config = {"offsets": [{"topic": topic, "partition": 0, "offset": 1}]}
def mock_edit_function(text=None, editor=None, env=None, require_save=None, extension=None, filename=None):
return yaml.dump(offset_config, default_flow_style=False)
monkeypatch.setattr(click, "edit", mock_edit_function)
result = interactive_cli_runner.invoke(
esque, args=["edit", "offsets", consumer_group, "-t", topic], input="y\n", catch_exceptions=False
)
assert result.exit_code == 0
# Check assertions:
consumergroup_desc_after = consumergroup_controller.get_consumer_group(consumer_id=consumer_group).describe(
partitions=True
)
assert consumergroup_desc_before["offsets"][topic][0]["consumer_offset"] == 10
assert consumergroup_desc_after["offsets"][topic][0]["consumer_offset"] == 1
def test_set_offsets_offset_from_group(
topic: str,
interactive_cli_runner,
producer: ConfluenceProducer,
consumer_group: str,
target_consumer_group: str,
consumergroup_controller: ConsumerGroupController,
):
produce_text_test_messages(producer=producer, topic_name=topic, amount=10)
consumergroup_controller.commit_offsets(
consumer_group, [TopicPartition(topic=topic, partition=0, offset=10)])
consumergroup_desc_before = consumergroup_controller.get_consumer_group(
consumer_id=consumer_group).describe(partitions=True)
interactive_cli_runner.invoke(
esque,
args=["set", "offsets", consumer_group, "--offset-by-delta", "-2"],
input="y\n",
catch_exceptions=False)
consumergroup_desc_after = consumergroup_controller.get_consumer_group(
consumer_id=consumer_group).describe(partitions=True)
# create a new consumer in a separate group and consume just one message
consumergroup_controller.commit_offsets(
target_consumer_group,
[TopicPartition(topic=topic, partition=0, offset=1)])
interactive_cli_runner.invoke(
esque,
args=[
"set", "offsets", target_consumer_group, "--offset-from-group",
consumer_group
],
input="y\n",
catch_exceptions=False,
)
consumergroup_desc_target = consumergroup_controller.get_consumer_group(
consumer_id=target_consumer_group).describe(partitions=True)
assert consumergroup_desc_before["offsets"][topic][0][
"consumer_offset"] == 10
assert consumergroup_desc_after["offsets"][topic][0][
"consumer_offset"] == 8
assert consumergroup_desc_target["offsets"][topic][0][
"consumer_offset"] == 8
def test_describe_topic_last_timestamp_does_not_commit(
non_interactive_cli_runner: CliRunner, topic: str,
consumergroup_controller: ConsumerGroupController, producer):
produce_text_test_messages(producer=producer, topic_name=topic, amount=10)
result = non_interactive_cli_runner.invoke(
esque,
args=["describe", "topic", topic, "--last-timestamp"],
catch_exceptions=False)
assert result.exit_code == 0
output = result.output
check_described_topic(output)
# cannot use pytest.raises(ConsumerGroupDoesNotExistException) because other tests may have committed offsets
# for this group
try:
data = consumergroup_controller.get_consumer_group(
config.ESQUE_GROUP_ID).describe(partitions=True)
assert topic.encode() not in data["offsets"]
except ConsumerGroupDoesNotExistException:
pass
def test_set_offsets_offset_to_timestamp_value(
topic: str,
interactive_cli_runner,
producer: ConfluenceProducer,
consumer_group: str,
consumergroup_controller: ConsumerGroupController,
):
messages = produce_text_test_messages(producer=producer,
topic_name=topic,
amount=10)
consumergroup_controller.commit_offsets(
consumer_group, [TopicPartition(topic=topic, partition=0, offset=10)])
consumergroup_desc_before = consumergroup_controller.get_consumer_group(
consumer_id=consumer_group).describe(partitions=True)
fifth_message = messages[4]
timestamp = fifth_message.timestamp
dt = pendulum.from_timestamp(round(timestamp / 1000) - 1)
interactive_cli_runner.invoke(
esque,
args=[
"set",
"offsets",
consumer_group,
"--topic-name",
topic,
"--offset-to-timestamp",
dt.format("YYYY-MM-DDTHH:mm:ss"),
],
input="y\n",
catch_exceptions=False,
)
# Check assertions:
consumergroup_desc_after = consumergroup_controller.get_consumer_group(
consumer_id=consumer_group).describe(partitions=True)
assert consumergroup_desc_before["offsets"][topic][0][
"consumer_offset"] == 10
assert consumergroup_desc_after["offsets"][topic][0][
"consumer_offset"] == 4
def test_transfer_plain_text_message_using_cli_pipe(
producer: ConfluentProducer,
target_topic_consumer: Consumer,
source_topic: Tuple[str, int],
target_topic: Tuple[str, int],
non_interactive_cli_runner: CliRunner,
):
expected_messages = produce_text_test_messages(topic_name=source_topic[0], producer=producer)
result1 = non_interactive_cli_runner.invoke(
esque, args=["consume", "--stdout", "--number", "10", source_topic[0]], catch_exceptions=False
)
non_interactive_cli_runner.invoke(
esque, args=["produce", "--stdin", target_topic[0]], input=result1.output, catch_exceptions=False
)
actual_messages = {
(msg.key().decode(), msg.value().decode(), msg.partition())
for msg in target_topic_consumer.consume(10, timeout=20)
}
expected_messages = {(msg.key, msg.value, msg.partition) for msg in expected_messages}
assert expected_messages == actual_messages
def test_transfer_plain_text_message_using_file(
producer: ConfluentProducer,
target_topic_consumer: Consumer,
source_topic: Tuple[str, int],
target_topic: Tuple[str, int],
non_interactive_cli_runner: CliRunner,
tmpdir_factory,
):
output_directory = tmpdir_factory.mktemp("output_directory")
expected_messages = produce_text_test_messages(topic_name=source_topic[0], producer=producer)
non_interactive_cli_runner.invoke(
esque, args=["consume", "-d", str(output_directory), "--number", "10", source_topic[0]], catch_exceptions=False
)
non_interactive_cli_runner.invoke(
esque, args=["produce", "-d", str(output_directory), target_topic[0]], catch_exceptions=False
)
actual_messages = {
(msg.key().decode(), msg.value().decode(), msg.partition())
for msg in target_topic_consumer.consume(10, timeout=20)
}
expected_messages = {(msg.key, msg.value, msg.partition) for msg in expected_messages}
assert expected_messages == actual_messages