def test_broker_decommission_empty_replication_group(self):
assignment = {
(u'T0', 0): ['0', '1', '2'],
(u'T0', 1): ['0', '1', '2'],
(u'T1', 0): ['0', '1'],
(u'T1', 1): ['1', '2'],
(u'T1', 2): ['1', '2'],
(u'T2', 0): ['0', '2'],
}
# r0 b0 t00, t01, t10, t20
# r1 b1 t00, t01, t10, t11, t12
# __ b2 t00, t01, t11, t12, t20 let's assume b2 is down and there is no
# metadata for it (it was in r2 before the failure)
# r2 b3 empty this broker came up to replace b2
brokers_rg = {'0': 'rg0', '1': 'rg1', '3': 'rg2'} # NOTE: b2 is not in this list
ct = ClusterTopology(
assignment,
brokers_rg,
lambda x: x.metadata, # The value of the broker dict is the metadata attribute
)
partitions_count = len(ct.partitions)
ct.decommission_brokers(['2'])
assert len(ct.partitions) == partitions_count
assert ct.brokers['2'].empty()
def test_broker_decommission_force(self):
assignment = {
(u'T0', 0): ['0', '1', '2'],
(u'T0', 1): ['0', '1', '2'],
(u'T1', 0): ['0', '1'],
(u'T1', 1): ['1', '4'],
(u'T1', 2): ['1', '4'],
(u'T2', 0): ['0', '3'],
}
# r1 b0 t00, t01, t10, t20
# r1 b1 t00, t01, t10, t11, t12
# r2 b2 t00, t01
# r2 b3 t20
# r3 b4 t11, t12
brokers_rg = {'0': 'rg1', '1': 'rg1', '2': 'rg2', '3': 'rg2', '4': 'r3'}
ct = ClusterTopology(
assignment,
brokers_rg,
lambda x: x.metadata, # The value of the broker dict is metadata
)
partitions_count = len(ct.partitions)
ct.decommission_brokers(['0'])
assert len(ct.partitions) == partitions_count
assert ct.brokers['0'].empty()
def display_cluster_topology_stats(cluster_topology, base_assignment=None):
if base_assignment:
base_cluster_topology = ClusterTopology(
base_assignment,
{
broker.id: broker.metadata
for broker in cluster_topology.brokers.values()
},
cluster_topology.partition_measurer,
lambda broker: cluster_topology.brokers[broker.id].
replication_group.id,
)
cluster_topologies = OrderedDict([
('Before', base_cluster_topology),
('After', cluster_topology),
])
else:
cluster_topologies = OrderedDict([
('', cluster_topology),
])
display_replica_imbalance(cluster_topologies)
print("")
display_partition_imbalance(cluster_topologies)
print("")
display_leader_imbalance(cluster_topologies)
print("")
display_topic_broker_imbalance(cluster_topologies)
if base_assignment:
print("")
display_movements_stats(cluster_topology, base_assignment)
def build_cluster_topology(self, assignment=None, brokers=None):
"""Create cluster topology from given assignment."""
if not assignment:
assignment = self._initial_assignment
if not brokers:
brokers = self.brokers
return ClusterTopology(assignment, brokers, self.get_replication_group_id)
def build_cluster_topology(assignment=None,
brokers=None,
get_replication_group_id=None,
partition_measurer=None):
assignment = assignment or default_assignment
brokers = brokers or default_brokers
get_replication_group_id = \
get_replication_group_id or default_get_replication_group_id
partition_measurer = partition_measurer or default_partition_measurer
return ClusterTopology(
assignment,
brokers,
partition_measurer,
get_replication_group_id,
)
def run(self, cluster_config, rg_parser, args):
self.cluster_config = cluster_config
self.args = args
with ZK(self.cluster_config) as self.zk:
self.log.debug(
'Starting %s for cluster: %s and zookeeper: %s',
self.__class__.__name__,
self.cluster_config.name,
self.cluster_config.zookeeper,
)
brokers = self.zk.get_brokers()
assignment = self.zk.get_cluster_assignment()
ct = ClusterTopology(
assignment,
brokers,
rg_parser.get_replication_group,
)
self.run_command(ct)
def run(
self,
cluster_config,
rg_parser,
partition_measurer,
cluster_balancer,
args,
):
"""Initialize cluster_config, args, and zk then call run_command."""
self.cluster_config = cluster_config
self.args = args
with ZK(self.cluster_config) as self.zk:
self.log.debug(
'Starting %s for cluster: %s and zookeeper: %s',
self.__class__.__name__,
self.cluster_config.name,
self.cluster_config.zookeeper,
)
brokers = self.zk.get_brokers()
assignment = self.zk.get_cluster_assignment()
pm = partition_measurer(
self.cluster_config,
brokers,
assignment,
args,
)
ct = ClusterTopology(
assignment,
brokers,
pm,
rg_parser.get_replication_group,
)
if len(ct.partitions) == 0:
self.log.info("The cluster is empty. No actions to perform.")
return
# Exit if there is an on-going reassignment
if self.is_reassignment_pending():
self.log.error('Previous reassignment pending.')
sys.exit(1)
self.run_command(ct, cluster_balancer(ct, args))
def test_cluster_topology_inactive_brokers(self):
assignment = {
(u'T0', 0): ['0', '1'],
(u'T0', 1): ['8', '9'], # 8 and 9 are not in active brokers
}
brokers = {
'0': {'host': 'host0'},
'1': {'host': 'host1'},
}
def extract_group(broker):
# group 0 for broker 0
# group 1 for broker 1
# None for inactive brokers
if broker in brokers:
return broker.id
return None
ct = ClusterTopology(assignment, brokers, extract_group)
assert ct.brokers['8'].inactive
assert ct.brokers['9'].inactive
assert None in ct.rgs
def zero_size_cluster_topology(orig_assignment):
""" This topology sets all of the partition sizes to be 0
"""
brokers = {0: None, 1: None, 2: None, 3: None}
pm = ZeroSizePartitionMeasurer({}, brokers, orig_assignment, {})
return ClusterTopology(orig_assignment, brokers, pm)
def orig_cluster_topology(orig_assignment):
""" This topology contains the original assignment
"""
brokers = {0: None, 1: None, 2: None, 3: None}
pm = UniformPartitionMeasurer({}, brokers, orig_assignment, {})
return ClusterTopology(orig_assignment, brokers, pm)
def empty_cluster_topology():
""" This topology contains an empty assignment
"""
brokers = {0: None, 1: None, 2: None, 3: None}
pm = UniformPartitionMeasurer({}, brokers, {}, {})
return ClusterTopology({}, brokers, pm)
def two_partition_same_topic_topology(two_partition_same_topic_assignment):
brokers = {0: None, 1: None, 2: None, 3: None}
pm = MixedSizePartitionMeasurer({}, brokers,
two_partition_same_topic_assignment, {})
return ClusterTopology(two_partition_same_topic_assignment, brokers, pm)
