def ping(state: State, times: int, wait: int):
"""Test the connection to the kafka cluster.
Ping the kafka cluster by writing messages to and reading messages from it.
After the specified number of "pings", return the minimum, maximum, and average time for the round trip.
"""
topic_controller = state.cluster.topic_controller
deltas = []
try:
try:
topic_controller.create_topics([Topic(PING_TOPIC)])
except TopicAlreadyExistsException:
pass
producer = PingProducer(PING_TOPIC)
consumer = ConsumerFactory().create_ping_consumer(group_id=PING_GROUP_ID, topic_name=PING_TOPIC)
click.echo(f"Pinging with {state.cluster.bootstrap_servers}.")
for i in range(times):
producer.produce()
_, delta = consumer.consume()
deltas.append(delta)
click.echo(f"m_seq={i} time={delta:.2f}ms")
sleep(wait)
except KeyboardInterrupt:
return
topic_controller.delete_topic(Topic(PING_TOPIC))
click.echo("--- statistics ---")
click.echo(f"{len(deltas)} messages sent/received.")
click.echo(f"min/avg/max = {min(deltas):.2f}/{(sum(deltas) / len(deltas)):.2f}/{max(deltas):.2f} ms")
def test_set_offsets_offset_to_delta(
topic: str,
produced_messages_same_partition,
interactive_cli_runner,
producer: ConfluenceProducer,
consumer_group,
consumergroup_controller,
tmpdir_factory,
):
produced_messages_same_partition(topic, producer)
vanilla_consumer = ConsumerFactory().create_consumer(
group_id=consumer_group,
topic_name=None,
output_directory=None,
last=False,
avro=False,
initialize_default_output_directory=False,
match=None,
enable_auto_commit=True,
)
vanilla_consumer.subscribe([topic])
vanilla_consumer.consume(10)
vanilla_consumer.close()
del vanilla_consumer
consumergroup_desc_before = consumergroup_controller.get_consumergroup(
consumer_id=consumer_group).describe(verbose=True)
interactive_cli_runner.invoke(
set_offsets,
args=[
consumer_group, "--topic-name", topic, "--offset-by-delta", "-2"
],
input="y\n",
catch_exceptions=True,
)
# Check assertions:
consumergroup_desc_after = consumergroup_controller.get_consumergroup(
consumer_id=consumer_group).describe(verbose=True)
assert consumergroup_desc_before["offsets"][topic.encode(
"UTF-8")][0]["consumer_offset"] == 10
assert consumergroup_desc_after["offsets"][topic.encode(
"UTF-8")][0]["consumer_offset"] == 8
def test_plain_text_consume_and_produce_newly_created_topic(
consumer_group: str,
interactive_cli_runner: CliRunner,
producer: ConfluenceProducer,
topic: str,
topic_id: str,
tmpdir_factory: TempdirFactory,
):
source_topic_id = topic
target_topic_id = topic_id
output_directory = tmpdir_factory.mktemp("output_directory")
produced_messages = produce_test_messages(producer, (source_topic_id, 1))
file_consumer = ConsumerFactory().create_consumer(consumer_group,
source_topic_id,
output_directory,
last=False,
avro=False)
file_consumer.consume(10)
result = interactive_cli_runner.invoke(
produce,
args=["-d", output_directory, target_topic_id],
input="y\n",
catch_exceptions=False)
assert result.exit_code == 0
# Check assertions:
assertion_check_directory = tmpdir_factory.mktemp(
"assertion_check_directory")
file_consumer = ConsumerFactory().create_consumer(
(consumer_group + "assertion_check"),
target_topic_id,
assertion_check_directory,
last=False,
avro=False)
file_consumer.consume(10)
consumed_messages = get_consumed_messages(assertion_check_directory, False)
assert produced_messages == consumed_messages
def test_avro_consume_and_produce(
consumer_group,
avro_producer: AvroProducer,
source_topic: Tuple[str, int],
target_topic: Tuple[str, int],
tmpdir_factory,
):
source_topic_id, _ = source_topic
target_topic_id, _ = target_topic
output_directory = tmpdir_factory.mktemp("output_directory")
produced_messages = produce_test_messages_with_avro(
avro_producer, source_topic)
file_consumer = ConsumerFactory().create_consumer(consumer_group,
source_topic_id,
output_directory,
False,
avro=True)
file_consumer.consume(10)
producer = ProducerFactory().create_producer(
topic_name=target_topic_id,
input_directory=output_directory,
avro=True)
producer.produce()
# Check assertions:
assertion_check_directory = tmpdir_factory.mktemp(
"assertion_check_directory")
file_consumer = ConsumerFactory().create_consumer(
consumer_group + "assertion_check",
target_topic_id,
assertion_check_directory,
False,
avro=True)
file_consumer.consume(10)
consumed_messages = get_consumed_messages(assertion_check_directory, True)
assert produced_messages == consumed_messages
def test_plain_text_consume_and_produce(
consumer_group,
producer: ConfluenceProducer,
source_topic: Tuple[str, int],
target_topic: Tuple[str, int],
tmpdir_factory,
):
source_topic_id, _ = source_topic
target_topic_id, _ = target_topic
output_directory = tmpdir_factory.mktemp("output_directory")
produced_messages = produce_test_messages(producer, source_topic)
file_consumer = ConsumerFactory().create_consumer(consumer_group,
source_topic_id,
output_directory,
False,
avro=False)
file_consumer.consume(10)
producer = ProducerFactory().create_producer(target_topic_id,
output_directory,
avro=False)
producer.produce()
# Check assertions:
assertion_check_directory = tmpdir_factory.mktemp(
"assertion_check_directory")
file_consumer = ConsumerFactory().create_consumer(
(consumer_group + "assertion_check"),
target_topic_id,
assertion_check_directory,
False,
avro=False)
file_consumer.consume(10)
consumed_messages = get_consumed_messages(assertion_check_directory, False)
assert produced_messages == consumed_messages
def test_avro_consume_to_file(consumer_group, avro_producer: AvroProducer,
source_topic: Tuple[str, int], tmpdir_factory):
source_topic_id, _ = source_topic
output_directory = tmpdir_factory.mktemp("output_directory")
produced_messages = produce_test_messages_with_avro(
avro_producer, source_topic)
file_consumer = ConsumerFactory().create_consumer(consumer_group,
source_topic_id,
output_directory,
False,
avro=True)
number_of_consumer_messages = file_consumer.consume(10)
consumed_messages = get_consumed_messages(output_directory, True)
assert number_of_consumer_messages == 10
assert produced_messages == consumed_messages
def test_plain_text_consume_to_file(consumer_group,
producer: ConfluenceProducer,
source_topic: Tuple[str,
int], tmpdir_factory):
source_topic_id, _ = source_topic
output_directory = tmpdir_factory.mktemp("output_directory")
produced_messages = produce_test_messages(producer, source_topic)
file_consumer = ConsumerFactory().create_consumer(consumer_group,
source_topic_id,
output_directory,
False,
avro=False)
number_of_consumer_messages = file_consumer.consume(10)
consumed_messages = get_consumed_messages(output_directory, False)
assert number_of_consumer_messages == 10
assert produced_messages == consumed_messages