def run_command(self, cluster_topology, cluster_balancer):
base_assignment = cluster_topology.assignment
cluster_balancer.revoke_leadership(self.args.broker_ids)
if not validate_plan(
assignment_to_plan(cluster_topology.assignment),
assignment_to_plan(base_assignment),
):
self.log.error('Invalid assignment %s.',
cluster_topology.assignment)
print(
'Invalid assignment: {0}'.format(cluster_topology.assignment),
file=sys.stderr,
)
sys.exit(1)
# Reduce the proposed assignment based on max_leader_changes
reduced_assignment = self.get_reduced_assignment(
base_assignment,
cluster_topology.assignment,
0, # Number of partition movements
self.args.max_leader_changes,
)
if reduced_assignment:
self.process_assignment(reduced_assignment)
else:
msg = "Cluster already balanced. No more partitions as leaders in " \
"revoked-leadership brokers."
self.log.info(msg)
print(msg)
def run_command(self, cluster_topology):
base_assignment = cluster_topology.assignment
cluster_topology.decommission_brokers(self.args.broker_ids)
if not validate_plan(
assignment_to_plan(cluster_topology.assignment),
assignment_to_plan(base_assignment),
):
self.log.error('Invalid assignment %s.', cluster_topology.assignment)
print(
'Invalid assignment: {0}'.format(cluster_topology.assignment),
file=sys.stderr,
)
sys.exit(1)
# Reduce the proposed assignment based on max_partition_movements
# and max_leader_changes
reduced_assignment = self.get_reduced_assignment(
base_assignment,
cluster_topology.assignment,
self.args.max_partition_movements,
self.args.max_leader_changes,
)
if reduced_assignment:
self.process_assignment(reduced_assignment)
else:
self.log.info(
"Cluster already balanced. No more replicas in "
"decommissioned brokers."
)
print(
"Cluster already balanced. No more replicas in "
"decommissioned brokers."
)
def run_command(self, cluster_topology):
if self.args.source_broker == self.args.dest_broker:
print("Error: Destination broker is same as source broker.")
sys.exit()
base_assignment = cluster_topology.assignment
cluster_topology.replace_broker(self.args.source_broker, self.args.dest_broker)
if not validate_plan(
assignment_to_plan(cluster_topology.assignment),
assignment_to_plan(base_assignment),
):
self.log.error('Invalid assignment %s.', cluster_topology.assignment)
print(
'Invalid assignment: {0}'.format(cluster_topology.assignment),
file=sys.stderr,
)
sys.exit(1)
# Reduce the proposed assignment based on max_partition_movements
# and max_leader_changes
reduced_assignment = self.get_reduced_assignment(
base_assignment,
cluster_topology.assignment,
self.args.max_partition_movements,
self.args.max_leader_changes,
)
if reduced_assignment:
self.process_assignment(reduced_assignment)
else:
self.log.info("Broker already replaced. No more replicas in source broker.")
print("Broker already replaced. No more replicas in source broker.")
def run_command(self, cluster_topology, cluster_balancer):
base_assignment = cluster_topology.assignment
cluster_balancer.decommission_brokers(self.args.broker_ids)
if not validate_plan(
assignment_to_plan(cluster_topology.assignment),
assignment_to_plan(base_assignment),
):
self.log.error('Invalid assignment %s.',
cluster_topology.assignment)
print(
'Invalid assignment: {0}'.format(cluster_topology.assignment),
file=sys.stderr,
)
sys.exit(1)
# Reduce the proposed assignment based on max_partition_movements
# and max_leader_changes
reduced_assignment = self.get_reduced_assignment(
base_assignment,
cluster_topology.assignment,
self.args.max_partition_movements,
self.args.max_leader_changes,
)
if reduced_assignment:
self.process_assignment(reduced_assignment)
else:
self.log.info("Cluster already balanced. No more replicas in "
"decommissioned brokers.")
print("Cluster already balanced. No more replicas in "
"decommissioned brokers.")
def run_command(self, cluster_topology, cluster_balancer):
base_assignment = cluster_topology.assignment
cluster_balancer.revoke_leadership(self.args.broker_ids)
if not validate_plan(
assignment_to_plan(cluster_topology.assignment),
assignment_to_plan(base_assignment),
):
self.log.error('Invalid assignment %s.', cluster_topology.assignment)
print(
'Invalid assignment: {0}'.format(cluster_topology.assignment),
file=sys.stderr,
)
sys.exit(1)
# Reduce the proposed assignment based on max_leader_changes
reduced_assignment = self.get_reduced_assignment(
base_assignment,
cluster_topology,
0, # Number of partition movements
self.args.max_leader_changes,
)
if reduced_assignment:
self.process_assignment(reduced_assignment)
else:
msg = "Cluster already balanced. No more partitions as leaders in " \
"revoked-leadership brokers."
self.log.info(msg)
print(msg)
def run_command(self, cluster_topology, cluster_balancer):
if self.args.source_broker == self.args.dest_broker:
print("Error: Destination broker is same as source broker.")
sys.exit()
if self.args.dest_broker is None:
self.log.warning('This will shrink the replica set of topics.')
base_assignment = cluster_topology.assignment
cluster_topology.replace_broker(self.args.source_broker,
self.args.dest_broker)
if not validate_plan(
assignment_to_plan(cluster_topology.assignment),
assignment_to_plan(base_assignment),
allow_rf_change=self.args.rf_change,
allow_rf_mismatch=self.args.rf_mismatch,
):
self.log.error('Invalid assignment %s.',
cluster_topology.assignment)
print(
'Invalid assignment: {0}'.format(cluster_topology.assignment),
file=sys.stderr,
)
sys.exit(1)
# Reduce the proposed assignment based on the topic_partition_filter, if provided
if self.args.topic_partition_filter:
self.log.info("Using provided filter list")
filter_set = self.get_topic_filter()
filtered_assignment = {}
for t_p, replica in six.iteritems(base_assignment):
if t_p in filter_set:
filtered_assignment[t_p] = replica
base_assignment = filtered_assignment
# Reduce the proposed assignment based on max_partition_movements
# and max_leader_changes
reduced_assignment = self.get_reduced_assignment(
base_assignment,
cluster_topology,
self.args.max_partition_movements,
self.args.max_leader_changes,
)
if reduced_assignment:
self.process_assignment(reduced_assignment,
allow_rf_change=self.args.rf_change,
allow_rf_mismatch=self.args.rf_mismatch)
else:
self.log.info(
"Broker already replaced. No more replicas in source broker.")
print(
"Broker already replaced. No more replicas in source broker.")
def run_command(self, ct):
plan_json = json.dumps(assignment_to_plan(ct.assignment))
if self.args.json_out:
with open(self.args.json_out, 'w') as f:
self.log.info(
'writing assignments as json to: %s',
self.args.json_out,
)
f.write(plan_json)
else:
self.log.info('writing assignments as json to stdout')
print plan_json
def run_command(self, cluster_topology, _):
plan_json = json.dumps(assignment_to_plan(cluster_topology.assignment))
if self.args.json_out:
with open(self.args.json_out, 'w') as f:
self.log.info(
'writing assignments as json to: %s',
self.args.json_out,
)
f.write(plan_json)
else:
self.log.info('writing assignments as json to stdout')
print(plan_json)
def run_command(self, cluster_topology, cluster_balancer):
if self.args.source_broker == self.args.dest_broker:
print("Error: Destination broker is same as source broker.")
sys.exit()
base_assignment = cluster_topology.assignment
cluster_topology.replace_broker(self.args.source_broker,
self.args.dest_broker)
if not validate_plan(
assignment_to_plan(cluster_topology.assignment),
assignment_to_plan(base_assignment),
):
self.log.error('Invalid assignment %s.',
cluster_topology.assignment)
print(
'Invalid assignment: {0}'.format(cluster_topology.assignment),
file=sys.stderr,
)
sys.exit(1)
# Reduce the proposed assignment based on max_partition_movements
# and max_leader_changes
reduced_assignment = self.get_reduced_assignment(
base_assignment,
cluster_topology.assignment,
self.args.max_partition_movements,
self.args.max_leader_changes,
)
if reduced_assignment:
self.process_assignment(reduced_assignment)
else:
self.log.info(
"Broker already replaced. No more replicas in source broker.")
print(
"Broker already replaced. No more replicas in source broker.")
def run_command(self, ct):
"""Get executable proposed plan(if any) for display or execution."""
base_assignment = ct.assignment
assignment = self.build_balanced_assignment(ct)
if not validate_plan(
assignment_to_plan(assignment),
assignment_to_plan(base_assignment),
):
self.log.error('Invalid latest-cluster assignment. Exiting.')
sys.exit(1)
# Reduce the proposed assignment based on max_partition_movements
# and max_leader_changes
reduced_assignment = self.get_reduced_assignment(
base_assignment,
assignment,
self.args.max_partition_movements,
self.args.max_leader_changes,
)
if reduced_assignment:
self.process_assignment(reduced_assignment)
else:
self.log.info("Cluster already balanced. No actions to perform.")
def run_command(self, ct):
"""Get executable proposed plan(if any) for display or execution."""
base_assignment = ct.assignment
assignment = self.build_balanced_assignment(ct)
if not validate_plan(
assignment_to_plan(assignment),
assignment_to_plan(base_assignment),
):
self.log.error('Invalid latest-cluster assignment. Exiting.')
sys.exit(1)
# Reduce the proposed assignment based on max_partition_movements
# and max_leader_changes
reduced_assignment = self.get_reduced_assignment(
base_assignment,
assignment,
self.args.max_partition_movements,
self.args.max_leader_changes,
)
if reduced_assignment:
self.process_assignment(reduced_assignment)
else:
self.log.info("Cluster already balanced. No actions to perform.")
def process_assignment(self, assignment, allow_rf_change=False):
plan = assignment_to_plan(assignment)
if self.args.proposed_plan_file:
self.log.info(
'Storing proposed-plan in %s',
self.args.proposed_plan_file,
)
self.write_json_plan(plan, self.args.proposed_plan_file)
self.log.info(
'Proposed plan assignment %s',
plan,
)
self.log.info(
'Proposed-plan actions count: %s',
len(plan['partitions']),
)
self.execute_plan(plan, allow_rf_change=allow_rf_change)
def process_assignment(self, assignment):
plan = assignment_to_plan(assignment)
if self.args.proposed_plan_file:
self.log.info(
'Storing proposed-plan in %s',
self.args.proposed_plan_file,
)
self.write_json_plan(plan, self.args.proposed_plan_file)
self.log.info(
'Proposed plan assignment %s',
plan,
)
self.log.info(
'Proposed-plan actions count: %s',
len(plan['partitions']),
)
self.execute_plan(plan)
def process_assignment(self,
assignment,
allow_rf_change=False,
allow_rf_mismatch=False):
plan = assignment_to_plan(assignment)
if self.args.proposed_plan_file:
self.log.info(
'Storing proposed-plan in %s',
self.args.proposed_plan_file,
)
self.write_json_plan(plan, self.args.proposed_plan_file)
self.log.info(
'Proposed plan assignment %s',
plan,
)
self.log.info(
'Proposed-plan actions count: %s',
len(plan['partitions']),
)
self.execute_plan(plan,
allow_rf_change=allow_rf_change,
allow_rf_mismatch=allow_rf_mismatch)
def run_command(self, cluster_topology, cluster_balancer):
# If the max_movement_size is still default, then the user did not input a value for it
if self.args.force_progress and self.args.max_movement_size == DEFAULT_MAX_MOVEMENT_SIZE:
self.log.error(
'--force-progress must be used with --max-movement-size', )
sys.exit(1)
# Obtain the largest partition in the set of partitions we will move
partitions_to_move = set()
for broker in self.args.broker_ids:
partitions_to_move.update(
cluster_topology.brokers[broker].partitions)
largest_size = max(partition.size for partition in partitions_to_move)
smallest_size = min(partition.size for partition in partitions_to_move)
if self.args.auto_max_movement_size:
self.args.max_movement_size = largest_size
self.log.info(
'Auto-max-movement-size: using {max_movement_size} as'
' max-movement-size.'.format(
max_movement_size=self.args.max_movement_size, ))
if self.args.max_movement_size and self.args.max_movement_size < largest_size:
if not self.args.force_progress:
self.log.error(
'Max partition movement size is only {max_movement_size},'
' but remaining partitions to move range from {smallest_size} to'
' {largest_size}. The decommission will not make progress'.
format(
max_movement_size=self.args.max_movement_size,
smallest_size=smallest_size,
largest_size=largest_size,
))
sys.exit(1)
else:
self.log.warning(
'Max partition movement size is only {max_movement_size},'
' but remaining partitions to move range from {smallest_size} to'
' {largest_size}. The decommission may be slower than expected'
.format(
max_movement_size=self.args.max_movement_size,
smallest_size=smallest_size,
largest_size=largest_size,
))
base_assignment = cluster_topology.assignment
cluster_balancer.decommission_brokers(self.args.broker_ids)
if not validate_plan(
assignment_to_plan(cluster_topology.assignment),
assignment_to_plan(base_assignment),
):
self.log.error('Invalid assignment %s.',
cluster_topology.assignment)
print(
'Invalid assignment: {0}'.format(cluster_topology.assignment),
file=sys.stderr,
)
sys.exit(1)
# Reduce the proposed assignment based on max_partition_movements
# and max_leader_changes
reduced_assignment = self.get_reduced_assignment(
base_assignment,
cluster_topology,
self.args.max_partition_movements,
self.args.max_leader_changes,
max_movement_size=self.args.max_movement_size,
force_progress=self.args.force_progress,
)
if reduced_assignment:
self.process_assignment(reduced_assignment)
else:
msg_str = "Cluster already balanced. No more replicas in decommissioned brokers."
self.log.info(msg_str)
print(msg_str)
def run_command(self, cluster_topology, cluster_balancer):
if self.args.force_progress and self.args.max_movement_size is None:
self.log.error(
'--force-progress must be used with --max-movement-size',
)
sys.exit(1)
# Obtain the largest partition in the set of partitions we will move
partitions_to_move = set()
for broker in self.args.broker_ids:
partitions_to_move.update(cluster_topology.brokers[broker].partitions)
largest_size = max(
partition.size
for partition in partitions_to_move
)
smallest_size = min(
partition.size
for partition in partitions_to_move
)
if self.args.auto_max_movement_size:
self.args.max_movement_size = largest_size
self.log.info(
'Auto-max-movement-size: using {max_movement_size} as'
' max-movement-size.'.format(
max_movement_size=self.args.max_movement_size,
)
)
if self.args.max_movement_size and self.args.max_movement_size < largest_size:
if not self.args.force_progress:
self.log.error(
'Max partition movement size is only {max_movement_size},'
' but remaining partitions to move range from {smallest_size} to'
' {largest_size}. The decommission will not make progress'.format(
max_movement_size=self.args.max_movement_size,
smallest_size=smallest_size,
largest_size=largest_size,
)
)
sys.exit(1)
else:
self.log.warning(
'Max partition movement size is only {max_movement_size},'
' but remaining partitions to move range from {smallest_size} to'
' {largest_size}. The decommission may be slower than expected'.format(
max_movement_size=self.args.max_movement_size,
smallest_size=smallest_size,
largest_size=largest_size,
)
)
base_assignment = cluster_topology.assignment
cluster_balancer.decommission_brokers(self.args.broker_ids)
if not validate_plan(
assignment_to_plan(cluster_topology.assignment),
assignment_to_plan(base_assignment),
):
self.log.error('Invalid assignment %s.', cluster_topology.assignment)
print(
'Invalid assignment: {0}'.format(cluster_topology.assignment),
file=sys.stderr,
)
sys.exit(1)
# Reduce the proposed assignment based on max_partition_movements
# and max_leader_changes
reduced_assignment = self.get_reduced_assignment(
base_assignment,
cluster_topology,
self.args.max_partition_movements,
self.args.max_leader_changes,
max_movement_size=self.args.max_movement_size,
force_progress=self.args.force_progress,
)
if reduced_assignment:
self.process_assignment(reduced_assignment)
else:
msg_str = "Cluster already balanced. No more replicas in decommissioned brokers."
self.log.info(msg_str)
print(msg_str)
def display_cluster_topology(cluster_topology):
print(assignment_to_plan(cluster_topology.assignment))
def display_cluster_topology(cluster_topology):
print(assignment_to_plan(cluster_topology.assignment))
def run_command(self, cluster_topology, cluster_balancer):
"""Get executable proposed plan(if any) for display or execution."""
# The ideal weight of each broker is total_weight / broker_count.
# It should be possible to remove partitions from each broker until
# the weight of the broker is less than this ideal value, otherwise it
# is impossible to balance the cluster. If --max-movement-size is too
# small, exit with an error.
if self.args.max_movement_size:
total_weight = sum(
partition.weight
for partition in cluster_topology.partitions.itervalues()
)
broker_count = len(cluster_topology.brokers)
optimal_weight = total_weight / broker_count
broker, max_unmovable_on_one_broker = max((
(broker, sum(
partition.weight
for partition in broker.partitions
if partition.size > self.args.max_movement_size
))
for broker in cluster_topology.brokers.values()),
key=lambda t: t[1],
)
if max_unmovable_on_one_broker >= optimal_weight:
sorted_partitions = sorted(
[
partition
for partition in broker.partitions
if partition.size > self.args.max_movement_size
],
reverse=True,
key=lambda partition: partition.size,
)
for partition in sorted_partitions:
max_unmovable_on_one_broker -= partition.weight
if max_unmovable_on_one_broker <= optimal_weight:
required_max_movement_size = partition.size
break
self.log.error(
'Max movement size {max_movement_size} is too small, it is'
' not be possible to balance the cluster. A max movement'
' size of {required} or higher is required.'.format(
max_movement_size=self.args.max_movement_size,
required=required_max_movement_size,
)
)
sys.exit(1)
elif self.args.auto_max_movement_size:
self.args.max_movement_size = max(
partition.size
for partition in cluster_topology.partitions.itervalues()
)
self.log.info(
'Auto-max-movement-size: using {max_movement_size} as'
' max-movement-size.'.format(
max_movement_size=self.args.max_movement_size,
)
)
base_assignment = cluster_topology.assignment
base_score = cluster_balancer.score()
rg_imbalance, _ = get_replication_group_imbalance_stats(
cluster_topology.rgs.values(),
cluster_topology.partitions.values()
)
cluster_balancer.rebalance()
assignment = cluster_topology.assignment
score = cluster_balancer.score()
new_rg_imbalance, _ = get_replication_group_imbalance_stats(
cluster_topology.rgs.values(),
cluster_topology.partitions.values()
)
if self.args.show_stats:
display_cluster_topology_stats(cluster_topology, base_assignment)
if base_score is not None and score is not None:
print('\nScore before: %f' % base_score)
print('Score after: %f' % score)
print('Score improvement: %f' % (score - base_score))
if not validate_plan(
assignment_to_plan(assignment),
assignment_to_plan(base_assignment),
):
self.log.error('Invalid latest-cluster assignment. Exiting.')
sys.exit(1)
if self.args.score_improvement_threshold:
if base_score is None or score is None:
self.log.error(
'%s cannot assign scores so --score-improvement-threshold'
' cannot be used.',
cluster_balancer.__class__.__name__,
)
return
else:
score_improvement = score - base_score
if score_improvement >= self.args.score_improvement_threshold:
self.log.info(
'Score improvement %f is greater than the threshold %f.'
' Continuing to apply the assignment.',
score_improvement,
self.args.score_improvement_threshold,
)
elif new_rg_imbalance < rg_imbalance:
self.log.info(
'Score improvement %f is less than the threshold %f,'
' but replica balance has improved. Continuing to'
' apply the assignment.',
score_improvement,
self.args.score_improvement_threshold,
)
else:
self.log.info(
'Score improvement %f is less than the threshold %f.'
' Assignment will not be applied.',
score_improvement,
self.args.score_improvement_threshold,
)
return
# Reduce the proposed assignment based on max_partition_movements
# and max_leader_changes
reduced_assignment = self.get_reduced_assignment(
base_assignment,
assignment,
self.args.max_partition_movements,
self.args.max_leader_changes,
)
if reduced_assignment:
self.process_assignment(reduced_assignment)
else:
self.log.info("Cluster already balanced. No actions to perform.")
def run_command(self, cluster_topology, cluster_balancer):
"""Get executable proposed plan(if any) for display or execution."""
# The ideal weight of each broker is total_weight / broker_count.
# It should be possible to remove partitions from each broker until
# the weight of the broker is less than this ideal value, otherwise it
# is impossible to balance the cluster. If --max-movement-size is too
# small, exit with an error.
if self.args.max_movement_size:
total_weight = sum(
partition.weight
for partition in six.itervalues(cluster_topology.partitions)
)
broker_count = len(cluster_topology.brokers)
optimal_weight = total_weight / broker_count
broker, max_unmovable_on_one_broker = max((
(broker, sum(
partition.weight
for partition in broker.partitions
if partition.size > self.args.max_movement_size
))
for broker in cluster_topology.brokers.values()),
key=lambda t: t[1],
)
if max_unmovable_on_one_broker >= optimal_weight:
sorted_partitions = sorted(
[
partition
for partition in broker.partitions
if partition.size > self.args.max_movement_size
],
reverse=True,
key=lambda partition: partition.size,
)
for partition in sorted_partitions:
max_unmovable_on_one_broker -= partition.weight
if max_unmovable_on_one_broker <= optimal_weight:
required_max_movement_size = partition.size
break
self.log.error(
'Max movement size {max_movement_size} is too small, it is'
' not be possible to balance the cluster. A max movement'
' size of {required} or higher is required.'.format(
max_movement_size=self.args.max_movement_size,
required=required_max_movement_size,
)
)
sys.exit(1)
elif self.args.auto_max_movement_size:
self.args.max_movement_size = max(
partition.size
for partition in six.itervalues(cluster_topology.partitions)
)
self.log.info(
'Auto-max-movement-size: using {max_movement_size} as'
' max-movement-size.'.format(
max_movement_size=self.args.max_movement_size,
)
)
base_assignment = cluster_topology.assignment
base_score = cluster_balancer.score()
rg_imbalance, _ = get_replication_group_imbalance_stats(
list(cluster_topology.rgs.values()),
list(cluster_topology.partitions.values())
)
cluster_balancer.rebalance()
assignment = cluster_topology.assignment
score = cluster_balancer.score()
new_rg_imbalance, _ = get_replication_group_imbalance_stats(
list(cluster_topology.rgs.values()),
list(cluster_topology.partitions.values())
)
if self.args.show_stats:
display_cluster_topology_stats(cluster_topology, base_assignment)
if base_score is not None and score is not None:
print('\nScore before: %f' % base_score)
print('Score after: %f' % score)
print('Score improvement: %f' % (score - base_score))
if not validate_plan(
assignment_to_plan(assignment),
assignment_to_plan(base_assignment),
):
self.log.error('Invalid latest-cluster assignment. Exiting.')
sys.exit(1)
if self.args.score_improvement_threshold:
if base_score is None or score is None:
self.log.error(
'%s cannot assign scores so --score-improvement-threshold'
' cannot be used.',
cluster_balancer.__class__.__name__,
)
return
else:
score_improvement = score - base_score
if score_improvement >= self.args.score_improvement_threshold:
self.log.info(
'Score improvement %f is greater than the threshold %f.'
' Continuing to apply the assignment.',
score_improvement,
self.args.score_improvement_threshold,
)
elif new_rg_imbalance < rg_imbalance:
self.log.info(
'Score improvement %f is less than the threshold %f,'
' but replica balance has improved. Continuing to'
' apply the assignment.',
score_improvement,
self.args.score_improvement_threshold,
)
else:
self.log.info(
'Score improvement %f is less than the threshold %f.'
' Assignment will not be applied.',
score_improvement,
self.args.score_improvement_threshold,
)
return
# Reduce the proposed assignment based on max_partition_movements
# and max_leader_changes
reduced_assignment = self.get_reduced_assignment(
base_assignment,
cluster_topology,
self.args.max_partition_movements,
self.args.max_leader_changes,
)
if reduced_assignment:
self.process_assignment(reduced_assignment)
else:
self.log.info("Cluster already balanced. No actions to perform.")
