def test_broker_decommission_empty_replication_group(self): assignment = { (u'T0', 0): ['0', '1', '2'], (u'T0', 1): ['0', '1', '2'], (u'T1', 0): ['0', '1'], (u'T1', 1): ['1', '2'], (u'T1', 2): ['1', '2'], (u'T2', 0): ['0', '2'], } # r0 b0 t00, t01, t10, t20 # r1 b1 t00, t01, t10, t11, t12 # __ b2 t00, t01, t11, t12, t20 let's assume b2 is down and there is no # metadata for it (it was in r2 before the failure) # r2 b3 empty this broker came up to replace b2 brokers_rg = {'0': 'rg0', '1': 'rg1', '3': 'rg2'} # NOTE: b2 is not in this list ct = ClusterTopology( assignment, brokers_rg, lambda x: x.metadata, # The value of the broker dict is the metadata attribute ) partitions_count = len(ct.partitions) ct.decommission_brokers(['2']) assert len(ct.partitions) == partitions_count assert ct.brokers['2'].empty()
def test_broker_decommission_force(self): assignment = { (u'T0', 0): ['0', '1', '2'], (u'T0', 1): ['0', '1', '2'], (u'T1', 0): ['0', '1'], (u'T1', 1): ['1', '4'], (u'T1', 2): ['1', '4'], (u'T2', 0): ['0', '3'], } # r1 b0 t00, t01, t10, t20 # r1 b1 t00, t01, t10, t11, t12 # r2 b2 t00, t01 # r2 b3 t20 # r3 b4 t11, t12 brokers_rg = {'0': 'rg1', '1': 'rg1', '2': 'rg2', '3': 'rg2', '4': 'r3'} ct = ClusterTopology( assignment, brokers_rg, lambda x: x.metadata, # The value of the broker dict is metadata ) partitions_count = len(ct.partitions) ct.decommission_brokers(['0']) assert len(ct.partitions) == partitions_count assert ct.brokers['0'].empty()
def display_cluster_topology_stats(cluster_topology, base_assignment=None): if base_assignment: base_cluster_topology = ClusterTopology( base_assignment, { broker.id: broker.metadata for broker in cluster_topology.brokers.values() }, cluster_topology.partition_measurer, lambda broker: cluster_topology.brokers[broker.id]. replication_group.id, ) cluster_topologies = OrderedDict([ ('Before', base_cluster_topology), ('After', cluster_topology), ]) else: cluster_topologies = OrderedDict([ ('', cluster_topology), ]) display_replica_imbalance(cluster_topologies) print("") display_partition_imbalance(cluster_topologies) print("") display_leader_imbalance(cluster_topologies) print("") display_topic_broker_imbalance(cluster_topologies) if base_assignment: print("") display_movements_stats(cluster_topology, base_assignment)
def build_cluster_topology(self, assignment=None, brokers=None): """Create cluster topology from given assignment.""" if not assignment: assignment = self._initial_assignment if not brokers: brokers = self.brokers return ClusterTopology(assignment, brokers, self.get_replication_group_id)
def build_cluster_topology(assignment=None, brokers=None, get_replication_group_id=None, partition_measurer=None): assignment = assignment or default_assignment brokers = brokers or default_brokers get_replication_group_id = \ get_replication_group_id or default_get_replication_group_id partition_measurer = partition_measurer or default_partition_measurer return ClusterTopology( assignment, brokers, partition_measurer, get_replication_group_id, )
def run(self, cluster_config, rg_parser, args): self.cluster_config = cluster_config self.args = args with ZK(self.cluster_config) as self.zk: self.log.debug( 'Starting %s for cluster: %s and zookeeper: %s', self.__class__.__name__, self.cluster_config.name, self.cluster_config.zookeeper, ) brokers = self.zk.get_brokers() assignment = self.zk.get_cluster_assignment() ct = ClusterTopology( assignment, brokers, rg_parser.get_replication_group, ) self.run_command(ct)
def run( self, cluster_config, rg_parser, partition_measurer, cluster_balancer, args, ): """Initialize cluster_config, args, and zk then call run_command.""" self.cluster_config = cluster_config self.args = args with ZK(self.cluster_config) as self.zk: self.log.debug( 'Starting %s for cluster: %s and zookeeper: %s', self.__class__.__name__, self.cluster_config.name, self.cluster_config.zookeeper, ) brokers = self.zk.get_brokers() assignment = self.zk.get_cluster_assignment() pm = partition_measurer( self.cluster_config, brokers, assignment, args, ) ct = ClusterTopology( assignment, brokers, pm, rg_parser.get_replication_group, ) if len(ct.partitions) == 0: self.log.info("The cluster is empty. No actions to perform.") return # Exit if there is an on-going reassignment if self.is_reassignment_pending(): self.log.error('Previous reassignment pending.') sys.exit(1) self.run_command(ct, cluster_balancer(ct, args))
def test_cluster_topology_inactive_brokers(self): assignment = { (u'T0', 0): ['0', '1'], (u'T0', 1): ['8', '9'], # 8 and 9 are not in active brokers } brokers = { '0': {'host': 'host0'}, '1': {'host': 'host1'}, } def extract_group(broker): # group 0 for broker 0 # group 1 for broker 1 # None for inactive brokers if broker in brokers: return broker.id return None ct = ClusterTopology(assignment, brokers, extract_group) assert ct.brokers['8'].inactive assert ct.brokers['9'].inactive assert None in ct.rgs
def zero_size_cluster_topology(orig_assignment): """ This topology sets all of the partition sizes to be 0 """ brokers = {0: None, 1: None, 2: None, 3: None} pm = ZeroSizePartitionMeasurer({}, brokers, orig_assignment, {}) return ClusterTopology(orig_assignment, brokers, pm)
def orig_cluster_topology(orig_assignment): """ This topology contains the original assignment """ brokers = {0: None, 1: None, 2: None, 3: None} pm = UniformPartitionMeasurer({}, brokers, orig_assignment, {}) return ClusterTopology(orig_assignment, brokers, pm)
def empty_cluster_topology(): """ This topology contains an empty assignment """ brokers = {0: None, 1: None, 2: None, 3: None} pm = UniformPartitionMeasurer({}, brokers, {}, {}) return ClusterTopology({}, brokers, pm)
def two_partition_same_topic_topology(two_partition_same_topic_assignment): brokers = {0: None, 1: None, 2: None, 3: None} pm = MixedSizePartitionMeasurer({}, brokers, two_partition_same_topic_assignment, {}) return ClusterTopology(two_partition_same_topic_assignment, brokers, pm)