def ping(state: State, times: int, wait: int):
"""Test the connection to the kafka cluster.
Ping the kafka cluster by writing messages to and reading messages from it.
After the specified number of "pings", return the minimum, maximum, and average time for the round trip.
"""
topic_controller = state.cluster.topic_controller
deltas = []
try:
try:
topic_controller.create_topics([Topic(PING_TOPIC)])
except TopicAlreadyExistsException:
pass
producer = PingProducer(PING_TOPIC)
consumer = ConsumerFactory().create_ping_consumer(group_id=PING_GROUP_ID, topic_name=PING_TOPIC)
click.echo(f"Pinging with {state.cluster.bootstrap_servers}.")
for i in range(times):
producer.produce()
_, delta = consumer.consume()
deltas.append(delta)
click.echo(f"m_seq={i} time={delta:.2f}ms")
sleep(wait)
except KeyboardInterrupt:
return
topic_controller.delete_topic(Topic(PING_TOPIC))
click.echo("--- statistics ---")
click.echo(f"{len(deltas)} messages sent/received.")
click.echo(f"min/avg/max = {min(deltas):.2f}/{(sum(deltas) / len(deltas)):.2f}/{max(deltas):.2f} ms")
def edit_consumer_group_offsets(
self, consumer_id: str,
offset_plan: List[ConsumerGroupOffsetPlan]):
"""
Commit consumergroup offsets to specific values
:param consumer_id: ID of the consumer group
:param offset_plan: List of ConsumerGroupOffsetPlan objects denoting the offsets for each partition in different topics
:return:
"""
consumer = ConsumerFactory().create_consumer(
group_id=consumer_id,
topic_name=None,
output_directory=None,
last=False,
avro=False,
initialize_default_output_directory=False,
match=None,
enable_auto_commit=False,
)
offsets = [
TopicPartition(topic=plan_element.topic_name,
partition=plan_element.partition_id,
offset=plan_element.proposed_offset)
for plan_element in offset_plan if not plan_element.offset_equal
]
consumer.commit(offsets=offsets)
def get_offsets_closest_to_timestamp(
self, topic_name: str,
timestamp_limit: pendulum) -> Dict[int, int]:
topic = self.get_cluster_topic(topic_name=topic_name)
partition_offsets = {
partition.partition_id: 0
for partition in topic.partitions
}
consumers = []
factory = ConsumerFactory()
partitions = partition_offsets.keys()
group_id = "group_for_" + topic_name + "_" + str(
int(round(time.time() * 1000)))
for partition in partitions:
consumer = factory.create_consumer(
group_id=group_id,
topic_name=None,
output_directory=None,
avro=False,
match=None,
last=False,
initialize_default_output_directory=False,
)
consumer.assign_specific_partitions(topic_name, [partition])
consumers.append(consumer)
for partition_counter in range(0, len(consumers)):
max_retry_count = 5
keep_polling_current_partition = True
while keep_polling_current_partition:
try:
message = consumers[
partition_counter].consume_single_message(timeout=10)
except MessageEmptyException:
# a possible timeout due to a network issue, retry (but not more than max_retry_count attempts)
max_retry_count -= 1
if max_retry_count <= 0:
keep_polling_current_partition = False
except EndOfPartitionReachedException:
keep_polling_current_partition = False
else:
if (datetime.datetime.fromtimestamp(
int(message.timestamp()[1] / 1000.0), timezone.utc)
< timestamp_limit):
partition_offsets[
message.partition()] = message.offset() + 1
return partition_offsets
def test_avro_consume_to_file(consumer_group, avro_producer: AvroProducer,
source_topic: Tuple[str, int], tmpdir_factory):
source_topic_id, _ = source_topic
output_directory = tmpdir_factory.mktemp("output_directory")
produced_messages = produce_test_messages_with_avro(
avro_producer, source_topic)
file_consumer = ConsumerFactory().create_consumer(consumer_group,
source_topic_id,
output_directory,
False,
avro=True)
number_of_consumer_messages = file_consumer.consume(10)
consumed_messages = get_consumed_messages(output_directory, True)
assert number_of_consumer_messages == 10
assert produced_messages == consumed_messages
def test_plain_text_consume_to_file(consumer_group,
producer: ConfluenceProducer,
source_topic: Tuple[str,
int], tmpdir_factory):
source_topic_id, _ = source_topic
output_directory = tmpdir_factory.mktemp("output_directory")
produced_messages = produce_test_messages(producer, source_topic)
file_consumer = ConsumerFactory().create_consumer(consumer_group,
source_topic_id,
output_directory,
False,
avro=False)
number_of_consumer_messages = file_consumer.consume(10)
consumed_messages = get_consumed_messages(output_directory, False)
assert number_of_consumer_messages == 10
assert produced_messages == consumed_messages
def test_set_offsets_offset_to_delta(
topic: str,
produced_messages_same_partition,
interactive_cli_runner,
producer: ConfluenceProducer,
consumer_group,
consumergroup_controller,
tmpdir_factory,
):
produced_messages_same_partition(topic, producer)
vanilla_consumer = ConsumerFactory().create_consumer(
group_id=consumer_group,
topic_name=None,
output_directory=None,
last=False,
avro=False,
initialize_default_output_directory=False,
match=None,
enable_auto_commit=True,
)
vanilla_consumer.subscribe([topic])
vanilla_consumer.consume(10)
vanilla_consumer.close()
del vanilla_consumer
consumergroup_desc_before = consumergroup_controller.get_consumergroup(
consumer_id=consumer_group).describe(verbose=True)
interactive_cli_runner.invoke(
set_offsets,
args=[
consumer_group, "--topic-name", topic, "--offset-by-delta", "-2"
],
input="y\n",
catch_exceptions=True,
)
# Check assertions:
consumergroup_desc_after = consumergroup_controller.get_consumergroup(
consumer_id=consumer_group).describe(verbose=True)
assert consumergroup_desc_before["offsets"][topic.encode(
"UTF-8")][0]["consumer_offset"] == 10
assert consumergroup_desc_after["offsets"][topic.encode(
"UTF-8")][0]["consumer_offset"] == 8
def test_plain_text_consume_and_produce_newly_created_topic(
consumer_group: str,
interactive_cli_runner: CliRunner,
producer: ConfluenceProducer,
topic: str,
topic_id: str,
tmpdir_factory: TempdirFactory,
):
source_topic_id = topic
target_topic_id = topic_id
output_directory = tmpdir_factory.mktemp("output_directory")
produced_messages = produce_test_messages(producer, (source_topic_id, 1))
file_consumer = ConsumerFactory().create_consumer(consumer_group,
source_topic_id,
output_directory,
last=False,
avro=False)
file_consumer.consume(10)
result = interactive_cli_runner.invoke(
produce,
args=["-d", output_directory, target_topic_id],
input="y\n",
catch_exceptions=False)
assert result.exit_code == 0
# Check assertions:
assertion_check_directory = tmpdir_factory.mktemp(
"assertion_check_directory")
file_consumer = ConsumerFactory().create_consumer(
(consumer_group + "assertion_check"),
target_topic_id,
assertion_check_directory,
last=False,
avro=False)
file_consumer.consume(10)
consumed_messages = get_consumed_messages(assertion_check_directory, False)
assert produced_messages == consumed_messages
def test_avro_consume_and_produce(
consumer_group,
avro_producer: AvroProducer,
source_topic: Tuple[str, int],
target_topic: Tuple[str, int],
tmpdir_factory,
):
source_topic_id, _ = source_topic
target_topic_id, _ = target_topic
output_directory = tmpdir_factory.mktemp("output_directory")
produced_messages = produce_test_messages_with_avro(
avro_producer, source_topic)
file_consumer = ConsumerFactory().create_consumer(consumer_group,
source_topic_id,
output_directory,
False,
avro=True)
file_consumer.consume(10)
producer = ProducerFactory().create_producer(
topic_name=target_topic_id,
input_directory=output_directory,
avro=True)
producer.produce()
# Check assertions:
assertion_check_directory = tmpdir_factory.mktemp(
"assertion_check_directory")
file_consumer = ConsumerFactory().create_consumer(
consumer_group + "assertion_check",
target_topic_id,
assertion_check_directory,
False,
avro=True)
file_consumer.consume(10)
consumed_messages = get_consumed_messages(assertion_check_directory, True)
assert produced_messages == consumed_messages
def test_plain_text_consume_and_produce(
consumer_group,
producer: ConfluenceProducer,
source_topic: Tuple[str, int],
target_topic: Tuple[str, int],
tmpdir_factory,
):
source_topic_id, _ = source_topic
target_topic_id, _ = target_topic
output_directory = tmpdir_factory.mktemp("output_directory")
produced_messages = produce_test_messages(producer, source_topic)
file_consumer = ConsumerFactory().create_consumer(consumer_group,
source_topic_id,
output_directory,
False,
avro=False)
file_consumer.consume(10)
producer = ProducerFactory().create_producer(target_topic_id,
output_directory,
avro=False)
producer.produce()
# Check assertions:
assertion_check_directory = tmpdir_factory.mktemp(
"assertion_check_directory")
file_consumer = ConsumerFactory().create_consumer(
(consumer_group + "assertion_check"),
target_topic_id,
assertion_check_directory,
False,
avro=False)
file_consumer.consume(10)
consumed_messages = get_consumed_messages(assertion_check_directory, False)
assert produced_messages == consumed_messages