class ReassignPartitionsTest(ProduceConsumeValidateTest):
"""
These tests validate partition reassignment.
Create a topic with few partitions, load some data, trigger partition re-assignment with and without broker failure,
check that partition re-assignment can complete and there is no data loss.
"""
def __init__(self, test_context):
""":type test_context: ducktape.tests.test.TestContext"""
super(ReassignPartitionsTest, self).__init__(test_context=test_context)
self.topic = "test_topic"
self.zk = ZookeeperService(test_context, num_nodes=1)
self.kafka = KafkaService(test_context,
num_nodes=4,
zk=self.zk,
topics={
self.topic: {
"partitions": 20,
"replication-factor": 3,
'configs': {
"min.insync.replicas": 2
}
}
})
self.num_partitions = 20
self.timeout_sec = 60
self.producer_throughput = 1000
self.num_producers = 1
self.num_consumers = 1
def setUp(self):
self.zk.start()
def min_cluster_size(self):
# Override this since we're adding services outside of the constructor
return super(
ReassignPartitionsTest,
self).min_cluster_size() + self.num_producers + self.num_consumers
def clean_bounce_some_brokers(self):
"""Bounce every other broker"""
for node in self.kafka.nodes[::2]:
self.kafka.restart_node(node, clean_shutdown=True)
def reassign_partitions(self, bounce_brokers):
partition_info = self.kafka.parse_describe_topic(
self.kafka.describe_topic(self.topic))
self.logger.debug("Partitions before reassignment:" +
str(partition_info))
# jumble partition assignment in dictionary
seed = random.randint(0, 2**31 - 1)
self.logger.debug("Jumble partition assignment with seed " + str(seed))
random.seed(seed)
# The list may still be in order, but that's ok
shuffled_list = range(0, self.num_partitions)
random.shuffle(shuffled_list)
for i in range(0, self.num_partitions):
partition_info["partitions"][i]["partition"] = shuffled_list[i]
self.logger.debug("Jumbled partitions: " + str(partition_info))
# send reassign partitions command
self.kafka.execute_reassign_partitions(partition_info)
if bounce_brokers:
# bounce a few brokers at the same time
self.clean_bounce_some_brokers()
# Wait until finished or timeout
wait_until(
lambda: self.kafka.verify_reassign_partitions(partition_info),
timeout_sec=self.timeout_sec,
backoff_sec=.5)
@cluster(num_nodes=7)
@parametrize(security_protocol="PLAINTEXT", bounce_brokers=True)
@parametrize(security_protocol="PLAINTEXT", bounce_brokers=False)
def test_reassign_partitions(self, bounce_brokers, security_protocol):
"""Reassign partitions tests.
Setup: 1 zk, 3 kafka nodes, 1 topic with partitions=3, replication-factor=3, and min.insync.replicas=2
- Produce messages in the background
- Consume messages in the background
- Reassign partitions
- If bounce_brokers is True, also bounce a few brokers while partition re-assignment is in progress
- When done reassigning partitions and bouncing brokers, stop producing, and finish consuming
- Validate that every acked message was consumed
"""
self.kafka.security_protocol = security_protocol
self.kafka.interbroker_security_protocol = security_protocol
new_consumer = False if self.kafka.security_protocol == "PLAINTEXT" else True
self.producer = VerifiableProducer(self.test_context,
self.num_producers,
self.kafka,
self.topic,
throughput=self.producer_throughput)
self.consumer = ConsoleConsumer(self.test_context,
self.num_consumers,
self.kafka,
self.topic,
new_consumer=new_consumer,
consumer_timeout_ms=60000,
message_validator=is_int)
self.kafka.start()
self.run_produce_consume_validate(
core_test_action=lambda: self.reassign_partitions(bounce_brokers))
class ThrottlingTest(ProduceConsumeValidateTest):
"""Tests throttled partition reassignment. This is essentially similar
to the reassign_partitions_test, except that we throttle the reassignment
and verify that it takes a sensible amount of time given the throttle
and the amount of data being moved.
Since the correctness is time dependent, this test also simplifies the
cluster topology. In particular, we fix the number of brokers, the
replication-factor, the number of partitions, the partition size, and
the number of partitions being moved so that we can accurately predict
the time throttled reassignment should take.
"""
def __init__(self, test_context):
""":type test_context: ducktape.tests.test.TestContext"""
super(ThrottlingTest, self).__init__(test_context=test_context)
self.topic = "test_topic"
self.zk = ZookeeperService(test_context, num_nodes=1)
# Because we are starting the producer/consumer/validate cycle _after_
# seeding the cluster with big data (to test throttling), we need to
# Start the consumer from the end of the stream. further, we need to
# ensure that the consumer is fully started before the producer starts
# so that we don't miss any messages. This timeout ensures the sufficient
# condition.
self.consumer_init_timeout_sec = 20
self.num_brokers = 6
self.num_partitions = 3
self.kafka = KafkaService(test_context,
num_nodes=self.num_brokers,
zk=self.zk,
topics={
self.topic: {
"partitions": self.num_partitions,
"replication-factor": 2,
"configs": {
"segment.bytes": 64 * 1024 * 1024
}
}
})
self.producer_throughput = 1000
self.timeout_sec = 400
self.num_records = 2000
self.record_size = 4096 * 100 # 400 KB
# 1 MB per partition on average.
self.partition_size = (self.num_records *
self.record_size) / self.num_partitions
self.num_producers = 2
self.num_consumers = 1
self.throttle = 4 * 1024 * 1024 # 4 MB/s
def setUp(self):
self.zk.start()
def min_cluster_size(self):
# Override this since we're adding services outside of the constructor
return super(ThrottlingTest, self).min_cluster_size() +\
self.num_producers + self.num_consumers
def clean_bounce_some_brokers(self):
"""Bounce every other broker"""
for node in self.kafka.nodes[::2]:
self.kafka.restart_node(node, clean_shutdown=True)
def reassign_partitions(self, bounce_brokers, throttle):
"""This method reassigns partitions using a throttle. It makes an
assertion about the minimum amount of time the reassignment should take
given the value of the throttle, the number of partitions being moved,
and the size of each partition.
"""
partition_info = self.kafka.parse_describe_topic(
self.kafka.describe_topic(self.topic))
self.logger.debug("Partitions before reassignment:" +
str(partition_info))
max_num_moves = 0
for i in range(0, self.num_partitions):
old_replicas = set(partition_info["partitions"][i]["replicas"])
new_part = (i + 1) % self.num_partitions
new_replicas = set(
partition_info["partitions"][new_part]["replicas"])
max_num_moves = max(len(new_replicas - old_replicas),
max_num_moves)
partition_info["partitions"][i]["partition"] = new_part
self.logger.debug("Jumbled partitions: " + str(partition_info))
self.kafka.execute_reassign_partitions(partition_info,
throttle=throttle)
start = time.time()
if bounce_brokers:
# bounce a few brokers at the same time
self.clean_bounce_some_brokers()
# Wait until finished or timeout
size_per_broker = max_num_moves * self.partition_size
self.logger.debug("Max amount of data transfer per broker: %fb",
size_per_broker)
estimated_throttled_time = math.ceil(
float(size_per_broker) / self.throttle)
estimated_time_with_buffer = estimated_throttled_time * 2
self.logger.debug("Waiting %ds for the reassignment to complete",
estimated_time_with_buffer)
wait_until(
lambda: self.kafka.verify_reassign_partitions(partition_info),
timeout_sec=estimated_time_with_buffer,
backoff_sec=.5)
stop = time.time()
time_taken = stop - start
self.logger.debug("Transfer took %d second. Estimated time : %ds",
time_taken, estimated_throttled_time)
assert time_taken >= estimated_throttled_time, \
("Expected rebalance to take at least %ds, but it took %ds" % (
estimated_throttled_time,
time_taken))
@cluster(num_nodes=10)
@parametrize(bounce_brokers=True)
@parametrize(bounce_brokers=False)
def test_throttled_reassignment(self, bounce_brokers):
security_protocol = 'PLAINTEXT'
self.kafka.security_protocol = security_protocol
self.kafka.interbroker_security_protocol = security_protocol
producer_id = 'bulk_producer'
bulk_producer = ProducerPerformanceService(
context=self.test_context,
num_nodes=1,
kafka=self.kafka,
topic=self.topic,
num_records=self.num_records,
record_size=self.record_size,
throughput=-1,
client_id=producer_id,
jmx_object_names=[
'kafka.producer:type=producer-metrics,client-id=%s' %
producer_id
],
jmx_attributes=['outgoing-byte-rate'])
self.producer = VerifiableProducer(context=self.test_context,
num_nodes=1,
kafka=self.kafka,
topic=self.topic,
message_validator=is_int,
throughput=self.producer_throughput)
self.consumer = ConsoleConsumer(self.test_context,
self.num_consumers,
self.kafka,
self.topic,
consumer_timeout_ms=60000,
message_validator=is_int,
from_beginning=False)
self.kafka.start()
bulk_producer.run()
self.run_produce_consume_validate(
core_test_action=lambda: self.reassign_partitions(
bounce_brokers, self.throttle))
class ReassignPartitionsTest(ProduceConsumeValidateTest):
"""
These tests validate partition reassignment.
Create a topic with few partitions, load some data, trigger partition re-assignment with and without broker failure,
check that partition re-assignment can complete and there is no data loss.
"""
def __init__(self, test_context):
""":type test_context: ducktape.tests.test.TestContext"""
super(ReassignPartitionsTest, self).__init__(test_context=test_context)
self.topic = "test_topic"
self.zk = ZookeeperService(test_context, num_nodes=1)
self.kafka = KafkaService(test_context, num_nodes=4, zk=self.zk, topics={self.topic: {
"partitions": 20,
"replication-factor": 3,
'configs': {"min.insync.replicas": 2}}
})
self.num_partitions = 20
self.timeout_sec = 60
self.producer_throughput = 1000
self.num_producers = 1
self.num_consumers = 1
def setUp(self):
self.zk.start()
def min_cluster_size(self):
# Override this since we're adding services outside of the constructor
return super(ReassignPartitionsTest, self).min_cluster_size() + self.num_producers + self.num_consumers
def clean_bounce_some_brokers(self):
"""Bounce every other broker"""
for node in self.kafka.nodes[::2]:
self.kafka.restart_node(node, clean_shutdown=True)
def reassign_partitions(self, bounce_brokers):
partition_info = self.kafka.parse_describe_topic(self.kafka.describe_topic(self.topic))
self.logger.debug("Partitions before reassignment:" + str(partition_info))
# jumble partition assignment in dictionary
seed = random.randint(0, 2 ** 31 - 1)
self.logger.debug("Jumble partition assignment with seed " + str(seed))
random.seed(seed)
# The list may still be in order, but that's ok
shuffled_list = range(0, self.num_partitions)
random.shuffle(shuffled_list)
for i in range(0, self.num_partitions):
partition_info["partitions"][i]["partition"] = shuffled_list[i]
self.logger.debug("Jumbled partitions: " + str(partition_info))
# send reassign partitions command
self.kafka.execute_reassign_partitions(partition_info)
if bounce_brokers:
# bounce a few brokers at the same time
self.clean_bounce_some_brokers()
# Wait until finished or timeout
wait_until(lambda: self.kafka.verify_reassign_partitions(partition_info), timeout_sec=self.timeout_sec, backoff_sec=.5)
@parametrize(security_protocol="PLAINTEXT", bounce_brokers=True)
@parametrize(security_protocol="PLAINTEXT", bounce_brokers=False)
def test_reassign_partitions(self, bounce_brokers, security_protocol):
"""Reassign partitions tests.
Setup: 1 zk, 3 kafka nodes, 1 topic with partitions=3, replication-factor=3, and min.insync.replicas=2
- Produce messages in the background
- Consume messages in the background
- Reassign partitions
- If bounce_brokers is True, also bounce a few brokers while partition re-assignment is in progress
- When done reassigning partitions and bouncing brokers, stop producing, and finish consuming
- Validate that every acked message was consumed
"""
self.kafka.security_protocol = security_protocol
self.kafka.interbroker_security_protocol = security_protocol
new_consumer = False if self.kafka.security_protocol == "PLAINTEXT" else True
self.producer = VerifiableProducer(self.test_context, self.num_producers, self.kafka, self.topic, throughput=self.producer_throughput)
self.consumer = ConsoleConsumer(self.test_context, self.num_consumers, self.kafka, self.topic, new_consumer=new_consumer, consumer_timeout_ms=60000, message_validator=is_int)
self.kafka.start()
self.run_produce_consume_validate(core_test_action=lambda: self.reassign_partitions(bounce_brokers))
class ReassignPartitionsTest(ProduceConsumeValidateTest):
"""
These tests validate partition reassignment.
Create a topic with few partitions, load some data, trigger partition re-assignment with and without broker failure,
check that partition re-assignment can complete and there is no data loss.
"""
def __init__(self, test_context):
""":type test_context: ducktape.tests.test.TestContext"""
super(ReassignPartitionsTest, self).__init__(test_context=test_context)
self.topic = "test_topic"
self.num_partitions = 20
self.zk = ZookeeperService(test_context, num_nodes=1)
# We set the min.insync.replicas to match the replication factor because
# it makes the test more stringent. If min.isr = 2 and
# replication.factor=3, then the test would tolerate the failure of
# reassignment for upto one replica per partition, which is not
# desirable for this test in particular.
self.kafka = KafkaService(test_context, num_nodes=4, zk=self.zk,
server_prop_overides=[
[config_property.LOG_ROLL_TIME_MS, "5000"],
[config_property.LOG_RETENTION_CHECK_INTERVAL_MS, "5000"]
],
topics={self.topic: {
"partitions": self.num_partitions,
"replication-factor": 3,
'configs': {
"min.insync.replicas": 3,
}}
})
self.timeout_sec = 60
self.producer_throughput = 1000
self.num_producers = 1
self.num_consumers = 1
def setUp(self):
self.zk.start()
def min_cluster_size(self):
# Override this since we're adding services outside of the constructor
return super(ReassignPartitionsTest, self).min_cluster_size() + self.num_producers + self.num_consumers
def clean_bounce_some_brokers(self):
"""Bounce every other broker"""
for node in self.kafka.nodes[::2]:
self.kafka.restart_node(node, clean_shutdown=True)
def reassign_partitions(self, bounce_brokers):
partition_info = self.kafka.parse_describe_topic(self.kafka.describe_topic(self.topic))
self.logger.debug("Partitions before reassignment:" + str(partition_info))
# jumble partition assignment in dictionary
seed = random.randint(0, 2 ** 31 - 1)
self.logger.debug("Jumble partition assignment with seed " + str(seed))
random.seed(seed)
# The list may still be in order, but that's ok
shuffled_list = range(0, self.num_partitions)
random.shuffle(shuffled_list)
for i in range(0, self.num_partitions):
partition_info["partitions"][i]["partition"] = shuffled_list[i]
self.logger.debug("Jumbled partitions: " + str(partition_info))
# send reassign partitions command
self.kafka.execute_reassign_partitions(partition_info)
if bounce_brokers:
# bounce a few brokers at the same time
self.clean_bounce_some_brokers()
# Wait until finished or timeout
wait_until(lambda: self.kafka.verify_reassign_partitions(partition_info),
timeout_sec=self.timeout_sec, backoff_sec=.5)
def move_start_offset(self):
"""We move the start offset of the topic by writing really old messages
and waiting for them to be cleaned up.
"""
producer = VerifiableProducer(self.test_context, 1, self.kafka, self.topic,
throughput=-1, enable_idempotence=True,
create_time=1000)
producer.start()
wait_until(lambda: producer.num_acked > 0,
timeout_sec=30,
err_msg="Failed to get an acknowledgement for %ds" % 30)
# Wait 8 seconds to let the topic be seeded with messages that will
# be deleted. The 8 seconds is important, since we should get 2 deleted
# segments in this period based on the configured log roll time and the
# retention check interval.
time.sleep(8)
producer.stop()
self.logger.info("Seeded topic with %d messages which will be deleted" %\
producer.num_acked)
# Since the configured check interval is 5 seconds, we wait another
# 6 seconds to ensure that at least one more cleaning so that the last
# segment is deleted. An altenate to using timeouts is to poll each
# partition until the log start offset matches the end offset. The
# latter is more robust.
time.sleep(6)
@cluster(num_nodes=8)
@matrix(bounce_brokers=[True, False],
reassign_from_offset_zero=[True, False])
def test_reassign_partitions(self, bounce_brokers, reassign_from_offset_zero):
"""Reassign partitions tests.
Setup: 1 zk, 4 kafka nodes, 1 topic with partitions=20, replication-factor=3,
and min.insync.replicas=3
- Produce messages in the background
- Consume messages in the background
- Reassign partitions
- If bounce_brokers is True, also bounce a few brokers while partition re-assignment is in progress
- When done reassigning partitions and bouncing brokers, stop producing, and finish consuming
- Validate that every acked message was consumed
"""
self.kafka.start()
if not reassign_from_offset_zero:
self.move_start_offset()
self.producer = VerifiableProducer(self.test_context, self.num_producers,
self.kafka, self.topic,
throughput=self.producer_throughput,
enable_idempotence=True)
self.consumer = ConsoleConsumer(self.test_context, self.num_consumers,
self.kafka, self.topic,
consumer_timeout_ms=60000,
message_validator=is_int)
self.enable_idempotence=True
self.run_produce_consume_validate(core_test_action=lambda: self.reassign_partitions(bounce_brokers))
class ReassignPartitionsTest(ProduceConsumeValidateTest):
"""
These tests validate partition reassignment.
Create a topic with few partitions, load some data, trigger partition re-assignment with and without broker failure,
check that partition re-assignment can complete and there is no data loss.
"""
def __init__(self, test_context):
""":type test_context: ducktape.tests.test.TestContext"""
super(ReassignPartitionsTest, self).__init__(test_context=test_context)
self.topic = "test_topic"
self.num_partitions = 20
self.zk = ZookeeperService(test_context, num_nodes=1)
# We set the min.insync.replicas to match the replication factor because
# it makes the test more stringent. If min.isr = 2 and
# replication.factor=3, then the test would tolerate the failure of
# reassignment for upto one replica per partition, which is not
# desirable for this test in particular.
self.kafka = KafkaService(
test_context,
num_nodes=4,
zk=self.zk,
server_prop_overides=[[
config_property.LOG_ROLL_TIME_MS, "5000"
], [config_property.LOG_RETENTION_CHECK_INTERVAL_MS, "5000"]],
topics={
self.topic: {
"partitions": self.num_partitions,
"replication-factor": 3,
'configs': {
"min.insync.replicas": 3,
}
}
})
self.timeout_sec = 60
self.producer_throughput = 1000
self.num_producers = 1
self.num_consumers = 1
def setUp(self):
self.zk.start()
def min_cluster_size(self):
# Override this since we're adding services outside of the constructor
return super(
ReassignPartitionsTest,
self).min_cluster_size() + self.num_producers + self.num_consumers
def clean_bounce_some_brokers(self):
"""Bounce every other broker"""
for node in self.kafka.nodes[::2]:
self.kafka.restart_node(node, clean_shutdown=True)
def reassign_partitions(self, bounce_brokers):
partition_info = self.kafka.parse_describe_topic(
self.kafka.describe_topic(self.topic))
self.logger.debug("Partitions before reassignment:" +
str(partition_info))
# jumble partition assignment in dictionary
seed = random.randint(0, 2**31 - 1)
self.logger.debug("Jumble partition assignment with seed " + str(seed))
random.seed(seed)
# The list may still be in order, but that's ok
shuffled_list = list(range(0, self.num_partitions))
random.shuffle(shuffled_list)
for i in range(0, self.num_partitions):
partition_info["partitions"][i]["partition"] = shuffled_list[i]
self.logger.debug("Jumbled partitions: " + str(partition_info))
# send reassign partitions command
self.kafka.execute_reassign_partitions(partition_info)
if bounce_brokers:
# bounce a few brokers at the same time
self.clean_bounce_some_brokers()
# Wait until finished or timeout
wait_until(
lambda: self.kafka.verify_reassign_partitions(partition_info),
timeout_sec=self.timeout_sec,
backoff_sec=.5)
def move_start_offset(self):
"""We move the start offset of the topic by writing really old messages
and waiting for them to be cleaned up.
"""
producer = VerifiableProducer(self.test_context,
1,
self.kafka,
self.topic,
throughput=-1,
enable_idempotence=True,
create_time=1000)
producer.start()
wait_until(lambda: producer.num_acked > 0,
timeout_sec=30,
err_msg="Failed to get an acknowledgement for %ds" % 30)
# Wait 8 seconds to let the topic be seeded with messages that will
# be deleted. The 8 seconds is important, since we should get 2 deleted
# segments in this period based on the configured log roll time and the
# retention check interval.
time.sleep(8)
producer.stop()
self.logger.info("Seeded topic with %d messages which will be deleted" %\
producer.num_acked)
# Since the configured check interval is 5 seconds, we wait another
# 6 seconds to ensure that at least one more cleaning so that the last
# segment is deleted. An altenate to using timeouts is to poll each
# partition until the log start offset matches the end offset. The
# latter is more robust.
time.sleep(6)
@cluster(num_nodes=8)
@matrix(bounce_brokers=[True, False],
reassign_from_offset_zero=[True, False])
def test_reassign_partitions(self, bounce_brokers,
reassign_from_offset_zero):
"""Reassign partitions tests.
Setup: 1 zk, 4 kafka nodes, 1 topic with partitions=20, replication-factor=3,
and min.insync.replicas=3
- Produce messages in the background
- Consume messages in the background
- Reassign partitions
- If bounce_brokers is True, also bounce a few brokers while partition re-assignment is in progress
- When done reassigning partitions and bouncing brokers, stop producing, and finish consuming
- Validate that every acked message was consumed
"""
self.kafka.start()
if not reassign_from_offset_zero:
self.move_start_offset()
self.producer = VerifiableProducer(self.test_context,
self.num_producers,
self.kafka,
self.topic,
throughput=self.producer_throughput,
enable_idempotence=True)
self.consumer = ConsoleConsumer(self.test_context,
self.num_consumers,
self.kafka,
self.topic,
consumer_timeout_ms=60000,
message_validator=is_int)
self.enable_idempotence = True
self.run_produce_consume_validate(
core_test_action=lambda: self.reassign_partitions(bounce_brokers))
class ThrottlingTest(ProduceConsumeValidateTest):
"""Tests throttled partition reassignment. This is essentially similar
to the reassign_partitions_test, except that we throttle the reassignment
and verify that it takes a sensible amount of time given the throttle
and the amount of data being moved.
Since the correctness is time dependent, this test also simplifies the
cluster topology. In particular, we fix the number of brokers, the
replication-factor, the number of partitions, the partition size, and
the number of partitions being moved so that we can accurately predict
the time throttled reassignment should take.
"""
def __init__(self, test_context):
""":type test_context: ducktape.tests.test.TestContext"""
super(ThrottlingTest, self).__init__(test_context=test_context)
self.topic = "test_topic"
self.zk = ZookeeperService(test_context, num_nodes=1)
# Because we are starting the producer/consumer/validate cycle _after_
# seeding the cluster with big data (to test throttling), we need to
# Start the consumer from the end of the stream. further, we need to
# ensure that the consumer is fully started before the producer starts
# so that we don't miss any messages. This timeout ensures the sufficient
# condition.
self.consumer_init_timeout_sec = 20
self.num_brokers = 6
self.num_partitions = 3
self.kafka = KafkaService(test_context,
num_nodes=self.num_brokers,
zk=self.zk,
topics={
self.topic: {
"partitions": self.num_partitions,
"replication-factor": 2,
"configs": {
"segment.bytes": 64 * 1024 * 1024
}
}
})
self.producer_throughput = 1000
self.timeout_sec = 400
self.num_records = 2000
self.record_size = 4096 * 100 # 400 KB
# 1 MB per partition on average.
self.partition_size = (self.num_records * self.record_size) / self.num_partitions
self.num_producers = 2
self.num_consumers = 1
self.throttle = 4 * 1024 * 1024 # 4 MB/s
def setUp(self):
self.zk.start()
def min_cluster_size(self):
# Override this since we're adding services outside of the constructor
return super(ThrottlingTest, self).min_cluster_size() +\
self.num_producers + self.num_consumers
def clean_bounce_some_brokers(self):
"""Bounce every other broker"""
for node in self.kafka.nodes[::2]:
self.kafka.restart_node(node, clean_shutdown=True)
def reassign_partitions(self, bounce_brokers, throttle):
"""This method reassigns partitions using a throttle. It makes an
assertion about the minimum amount of time the reassignment should take
given the value of the throttle, the number of partitions being moved,
and the size of each partition.
"""
partition_info = self.kafka.parse_describe_topic(
self.kafka.describe_topic(self.topic))
self.logger.debug("Partitions before reassignment:" +
str(partition_info))
max_num_moves = 0
for i in range(0, self.num_partitions):
old_replicas = set(partition_info["partitions"][i]["replicas"])
new_part = (i+1) % self.num_partitions
new_replicas = set(partition_info["partitions"][new_part]["replicas"])
max_num_moves = max(len(new_replicas - old_replicas), max_num_moves)
partition_info["partitions"][i]["partition"] = new_part
self.logger.debug("Jumbled partitions: " + str(partition_info))
self.kafka.execute_reassign_partitions(partition_info,
throttle=throttle)
start = time.time()
if bounce_brokers:
# bounce a few brokers at the same time
self.clean_bounce_some_brokers()
# Wait until finished or timeout
size_per_broker = max_num_moves * self.partition_size
self.logger.debug("Max amount of data transfer per broker: %fb",
size_per_broker)
estimated_throttled_time = math.ceil(float(size_per_broker) /
self.throttle)
estimated_time_with_buffer = estimated_throttled_time * 2
self.logger.debug("Waiting %ds for the reassignment to complete",
estimated_time_with_buffer)
wait_until(lambda: self.kafka.verify_reassign_partitions(partition_info),
timeout_sec=estimated_time_with_buffer, backoff_sec=.5)
stop = time.time()
time_taken = stop - start
self.logger.debug("Transfer took %d second. Estimated time : %ds",
time_taken,
estimated_throttled_time)
assert time_taken >= estimated_throttled_time * 0.9, \
("Expected rebalance to take at least %ds, but it took %ds" % (
estimated_throttled_time,
time_taken))
@cluster(num_nodes=10)
@parametrize(bounce_brokers=True)
@parametrize(bounce_brokers=False)
def test_throttled_reassignment(self, bounce_brokers):
security_protocol = 'PLAINTEXT'
self.kafka.security_protocol = security_protocol
self.kafka.interbroker_security_protocol = security_protocol
producer_id = 'bulk_producer'
bulk_producer = ProducerPerformanceService(
context=self.test_context, num_nodes=1, kafka=self.kafka,
topic=self.topic, num_records=self.num_records,
record_size=self.record_size, throughput=-1, client_id=producer_id)
self.producer = VerifiableProducer(context=self.test_context,
num_nodes=1,
kafka=self.kafka, topic=self.topic,
message_validator=is_int,
throughput=self.producer_throughput)
self.consumer = ConsoleConsumer(self.test_context,
self.num_consumers,
self.kafka,
self.topic,
consumer_timeout_ms=60000,
message_validator=is_int,
from_beginning=False)
self.kafka.start()
bulk_producer.run()
self.run_produce_consume_validate(core_test_action=
lambda: self.reassign_partitions(bounce_brokers, self.throttle))
self.logger.debug("Bulk producer outgoing-byte-rates: %s",
(metric.value for k, metrics in
bulk_producer.metrics(group='producer-metrics', name='outgoing-byte-rate', client_id=producer_id) for
metric in metrics)
)
