def test_rebalance_leaders_unbalanced_case3( self, create_balancer, create_cluster_topology, ): # Imbalanced 0 and 2. No re-balance possible. assignment = dict( [ ((u'T1', 0), ['1', '2']), ((u'T1', 1), ['0']), ((u'T2', 0), ['0']), ] ) ct = create_cluster_topology(assignment, broker_range(3)) cb = create_balancer(ct) cb.rebalance_leaders() # Verify still leader-imbalanced leader_imbal = get_net_imbalance( get_broker_leader_counts(ct.brokers.values()), ) assert leader_imbal == 1 # No change in assignment assert sorted(ct.assignment) == sorted(assignment)
def test_revoke_leadership_single_broker( self, create_balancer, create_cluster_topology, ): assignment = dict([ ((u'T0', 0), ['2', '0']), ((u'T1', 0), ['2', '1']), ((u'T1', 1), ['0', '2']), ((u'T2', 0), ['1', '0']), ]) ct = create_cluster_topology(assignment, broker_range(3)) cb = create_balancer(ct) cb.revoke_leadership(['2']) new_leaders_per_broker = { broker.id: broker.count_preferred_replica() for broker in six.itervalues(ct.brokers) } _, total_movements = \ calculate_partition_movement(assignment, ct.assignment) # Get net imbalance statistics excluding brokers to be revoked # leadership from brokers = [ b for b in six.itervalues(ct.brokers) if b.id not in ['2', '3'] ] new_net_imbal = get_net_imbalance(get_broker_leader_counts(brokers)) # Verify that broker '2' is not leader of any partition assert new_leaders_per_broker['2'] == 0 assert new_leaders_per_broker['1'] == 2 assert new_leaders_per_broker['0'] == 2 # Assert no partition movements assert total_movements == 0 # Assert remaining brokers are balanced with leader count assert new_net_imbal == 0
def test_rebalance_leaders_unbalanced_case2( self, create_balancer, create_cluster_topology, ): # (Broker: leader-count): {0: 2, 1: 1, 2:0} # opt-count: 3/3 = 1, extra-count = 0 # Leader-imbalance-value: 1 assignment = dict( [ ((u'T0', 0), ['1', '2']), ((u'T1', 1), ['0', '1']), ((u'T1', 0), ['0']), ] ) ct = create_cluster_topology(assignment, broker_range(3)) cb = create_balancer(ct) cb.rebalance_leaders() # Verify leader-balanced leader_imbal = get_net_imbalance( get_broker_leader_counts(ct.brokers.values()), ) assert leader_imbal == 0
def test_rebalance_leaders_balanced_case1( self, create_balancer, create_cluster_topology, ): # Already balanced-assignment with evenly-distributed # (broker-id: leader-count): {0: 1, 1:1, 2:1} # opt-count: 3/3 = 1, extra-count: 3%3 = 0 assignment = dict([ ((u'T0', 0), ['1', '2']), ((u'T0', 1), ['2', '0']), ((u'T1', 0), ['0', '2']), ]) ct = create_cluster_topology(assignment, broker_range(3)) orig_assignment = ct.assignment cb = create_balancer(ct) cb.rebalance_leaders() net_imbal = get_net_imbalance( get_broker_leader_counts(list(ct.brokers.values())), ) # No changed in already-balanced assignment assert orig_assignment == ct.assignment # Assert leader-balanced assert net_imbal == 0
def test_rebalance_leaders_unbalanced_case5( self, create_balancer, create_cluster_topology, ): # Special case, wherein none under-balanced # but 0 is overbalanced assignment = dict( [ ((u'T1', 1), ['0', '1']), ((u'T2', 0), ['0']), ((u'T2', 1), ['0']), ((u'T3', 0), ['2', '3']), ((u'T3', 1), ['3', '1']), ((u'T4', 0), ['1']), ] ) ct = create_cluster_topology(assignment, broker_range(4)) cb = create_balancer(ct) cb.rebalance_leaders() # Verify leader-balanced leader_imbal = get_net_imbalance( get_broker_leader_counts(ct.brokers.values()), ) assert leader_imbal == 0
def test_rebalance_leaders_unbalanced_case1( self, create_balancer, create_cluster_topology, ): # Balance leader-imbalance successfully # (broker-id: leader-count): {0: 0, 1:2, 2:1} # Net-leader-imbalance: 1 # opt-count: 3/3 = 1, extra-count: 3%3 = 0 assignment = dict([ ((u'T0', 0), ['1', '2']), ((u'T0', 1), ['2', '0']), ((u'T1', 0), ['1', '0']), ]) ct = create_cluster_topology(assignment, broker_range(3)) orig_assignment = ct.assignment cb = create_balancer(ct) cb.rebalance_leaders() # Verify if valid-leader assignment self.assert_leader_valid(orig_assignment, ct.assignment) # New-leader imbalance-count be less than previous imbal count new_leaders_per_broker = { broker.id: broker.count_preferred_replica() for broker in six.itervalues(ct.brokers) } new_leader_imbal = get_net_imbalance( list(new_leaders_per_broker.values())) # Verify leader-balanced assert new_leader_imbal == 0 # Verify partitions-changed assignment assert new_leaders_per_broker['0'] == 1 assert new_leaders_per_broker['1'] == 1 assert new_leaders_per_broker['2'] == 1
def test_rebalance_leaders_unbalanced_case3( self, create_balancer, create_cluster_topology, ): # Imbalanced 0 and 2. No re-balance possible. assignment = dict( [ ((u'T1', 0), ['1', '2']), ((u'T1', 1), ['0']), ((u'T2', 0), ['0']), ] ) ct = create_cluster_topology(assignment, broker_range(3)) cb = create_balancer(ct) cb.rebalance_leaders() # Verify still leader-imbalanced leader_imbal = get_net_imbalance( get_broker_leader_counts(list(ct.brokers.values())), ) assert leader_imbal == 1 # No change in assignment assert sorted(ct.assignment) == sorted(assignment)
def test_rebalance_leaders_unbalanced_case2d( self, create_balancer, create_cluster_topology, ): # Broker-2 imbalanced with same brokers # Broker-2 requests leadership from same broker-1 twice assignment = dict( [ ((u'T1', 0), ['1', '2']), ((u'T1', 1), ['0', '1']), ((u'T1', 2), ['0']), ((u'T1', 3), ['1', '2']), ((u'T1', 4), ['0', '1']), ((u'T1', 5), ['0']), ] ) ct = create_cluster_topology(assignment, broker_range(3)) cb = create_balancer(ct) cb.rebalance_leaders() # Verify leader-balanced leader_imbal = get_net_imbalance( get_broker_leader_counts(ct.brokers.values()), ) assert leader_imbal == 0
def test_rebalance_leaders_unbalanced_case2d( self, create_balancer, create_cluster_topology, ): # Broker-2 imbalanced with same brokers # Broker-2 requests leadership from same broker-1 twice assignment = dict( [ ((u'T1', 0), ['1', '2']), ((u'T1', 1), ['0', '1']), ((u'T1', 2), ['0']), ((u'T1', 3), ['1', '2']), ((u'T1', 4), ['0', '1']), ((u'T1', 5), ['0']), ] ) ct = create_cluster_topology(assignment, broker_range(3)) cb = create_balancer(ct) cb.rebalance_leaders() # Verify leader-balanced leader_imbal = get_net_imbalance( get_broker_leader_counts(list(ct.brokers.values())), ) assert leader_imbal == 0
def test_rebalance_leaders_unbalanced_case2e( self, create_balancer, create_cluster_topology, ): # Imbalance-val 2 # Multiple imbalanced brokers (2, 5) gets non-follower balanced # from multiple brokers (1,4) assignment = dict( [ ((u'T1', 0), ['1', '2']), ((u'T1', 1), ['0', '1']), ((u'T2', 0), ['0']), ((u'T3', 0), ['4', '5']), ((u'T3', 1), ['3', '4']), ((u'T4', 0), ['3']), ] ) ct = create_cluster_topology(assignment, broker_range(6)) cb = create_balancer(ct) cb.rebalance_leaders() # Verify leader-balanced leader_imbal = get_net_imbalance( get_broker_leader_counts(list(ct.brokers.values())), ) assert leader_imbal == 0
def test_rebalance_leaders_unbalanced_case2a( self, create_balancer, create_cluster_topology, ): # (Broker: leader-count): {0: 2, 1: 1, 2:0, 3:1} # opt-count: 3/4 = 1, extra-count = 3 # Leader-imbalance-value: 1 # imbalanced-broker: 0,2; balanced-brokers: 1,3 assignment = dict([ ((u'T0', 0), ['3', '2']), ((u'T0', 1), ['1', '3']), ((u'T1', 1), ['0', '1']), ((u'T1', 0), ['0']), ]) ct = create_cluster_topology(assignment, broker_range(4)) cb = create_balancer(ct) cb.rebalance_leaders() # Verify balanced leader_imbal = get_net_imbalance( get_broker_leader_counts(list(ct.brokers.values())), ) assert leader_imbal == 0 # Verify that (T0, 1) also swapped even if 1 and 3 were balanced # Rebalancing through non-followers replica_ids = [b.id for b in ct.partitions[('T0', 1)].replicas] assert replica_ids == ['3', '1']
def test_rebalance_leaders_unbalanced_case2c( self, create_balancer, create_cluster_topology, ): # Broker-2 imbalance value: 2 with different brokers # Broker-2 requests leadership from multiple brokers (0, 1) once assignment = dict( [ ((u'T1', 0), ['1', '2']), ((u'T1', 1), ['0', '1']), ((u'T2', 0), ['0']), ((u'T2', 1), ['0']), ((u'T3', 0), ['3', '2']), ((u'T3', 1), ['1', '3']), ((u'T4', 0), ['1']), ((u'T4', 2), ['3']), ] ) ct = create_cluster_topology(assignment, broker_range(4)) cb = create_balancer(ct) cb.rebalance_leaders() # Verify leader-balanced leader_imbal = get_net_imbalance( get_broker_leader_counts(list(ct.brokers.values())), ) assert leader_imbal == 0
def test_rebalance_leaders_unbalanced_case2( self, create_balancer, create_cluster_topology, ): # (Broker: leader-count): {0: 2, 1: 1, 2:0} # opt-count: 3/3 = 1, extra-count = 0 # Leader-imbalance-value: 1 assignment = dict( [ ((u'T0', 0), ['1', '2']), ((u'T1', 1), ['0', '1']), ((u'T1', 0), ['0']), ] ) ct = create_cluster_topology(assignment, broker_range(3)) cb = create_balancer(ct) cb.rebalance_leaders() # Verify leader-balanced leader_imbal = get_net_imbalance( get_broker_leader_counts(list(ct.brokers.values())), ) assert leader_imbal == 0
def test_rebalance_leaders_unbalanced_case2a( self, create_balancer, create_cluster_topology, ): # (Broker: leader-count): {0: 2, 1: 1, 2:0, 3:1} # opt-count: 3/4 = 1, extra-count = 3 # Leader-imbalance-value: 1 # imbalanced-broker: 0,2; balanced-brokers: 1,3 assignment = dict( [ ((u'T0', 0), ['3', '2']), ((u'T0', 1), ['1', '3']), ((u'T1', 1), ['0', '1']), ((u'T1', 0), ['0']), ] ) ct = create_cluster_topology(assignment, broker_range(4)) cb = create_balancer(ct) cb.rebalance_leaders() # Verify balanced leader_imbal = get_net_imbalance( get_broker_leader_counts(list(ct.brokers.values())), ) assert leader_imbal == 0 # Verify that (T0, 1) also swapped even if 1 and 3 were balanced # Rebalancing through non-followers replica_ids = [b.id for b in ct.partitions[('T0', 1)].replicas] assert replica_ids == ['3', '1']
def test_rebalance_leaders_unbalanced_case2c( self, create_balancer, create_cluster_topology, ): # Broker-2 imbalance value: 2 with different brokers # Broker-2 requests leadership from multiple brokers (0, 1) once assignment = dict([ ((u'T1', 0), ['1', '2']), ((u'T1', 1), ['0', '1']), ((u'T2', 0), ['0']), ((u'T2', 1), ['0']), ((u'T3', 0), ['3', '2']), ((u'T3', 1), ['1', '3']), ((u'T4', 0), ['1']), ((u'T4', 2), ['3']), ]) ct = create_cluster_topology(assignment, broker_range(4)) cb = create_balancer(ct) cb.rebalance_leaders() # Verify leader-balanced leader_imbal = get_net_imbalance( get_broker_leader_counts(list(ct.brokers.values())), ) assert leader_imbal == 0
def test_rebalance_leaders_balanced_case2( self, create_balancer, create_cluster_topology, ): # Already balanced-assignment NOT evenly-distributed # (broker-id: leader-count): {0: 1, 1:1, 2:1} # opt-count: 2/3 = 0, extra-count: 2%3 = 2 assignment = dict( [ ((u'T0', 0), ['1', '2']), ((u'T0', 1), ['2', '0']), ] ) ct = create_cluster_topology(assignment, broker_range(3)) orig_assignment = ct.assignment cb = create_balancer(ct) cb.rebalance_leaders() net_imbal = get_net_imbalance( get_broker_leader_counts(list(ct.brokers.values())), ) # No changed in already-balanced assignment assert orig_assignment == ct.assignment # Assert leader-balanced assert net_imbal == 0
def test_rebalance_leaders_unbalanced_case2e( self, create_balancer, create_cluster_topology, ): # Imbalance-val 2 # Multiple imbalanced brokers (2, 5) gets non-follower balanced # from multiple brokers (1,4) assignment = dict([ ((u'T1', 0), ['1', '2']), ((u'T1', 1), ['0', '1']), ((u'T2', 0), ['0']), ((u'T3', 0), ['4', '5']), ((u'T3', 1), ['3', '4']), ((u'T4', 0), ['3']), ]) ct = create_cluster_topology(assignment, broker_range(6)) cb = create_balancer(ct) cb.rebalance_leaders() # Verify leader-balanced leader_imbal = get_net_imbalance( get_broker_leader_counts(list(ct.brokers.values())), ) assert leader_imbal == 0
def test_rebalance_leaders_unbalanced_case5( self, create_balancer, create_cluster_topology, ): # Special case, wherein none under-balanced # but 0 is overbalanced assignment = dict( [ ((u'T1', 1), ['0', '1']), ((u'T2', 0), ['0']), ((u'T2', 1), ['0']), ((u'T3', 0), ['2', '3']), ((u'T3', 1), ['3', '1']), ((u'T4', 0), ['1']), ] ) ct = create_cluster_topology(assignment, broker_range(4)) cb = create_balancer(ct) cb.rebalance_leaders() # Verify leader-balanced leader_imbal = get_net_imbalance( get_broker_leader_counts(list(ct.brokers.values())), ) assert leader_imbal == 0
def display_leader_imbalance(cluster_topologies): """Display leader count and weight imbalance statistics. :param cluster_topologies: A dictionary mapping a string name to a ClusterTopology object. """ broker_ids = list(next(six.itervalues(cluster_topologies)).brokers.keys()) assert all( set(broker_ids) == set(cluster_topology.brokers.keys()) for cluster_topology in six.itervalues(cluster_topologies)) broker_leader_counts = [ stats.get_broker_leader_counts(cluster_topology.brokers[broker_id] for broker_id in broker_ids) for cluster_topology in six.itervalues(cluster_topologies) ] broker_leader_weights = [ stats.get_broker_leader_weights(cluster_topology.brokers[broker_id] for broker_id in broker_ids) for cluster_topology in six.itervalues(cluster_topologies) ] _display_table_title_multicolumn( 'Leader Count', 'Brokers', broker_ids, list(cluster_topologies.keys()), broker_leader_counts, ) print('') _display_table_title_multicolumn( 'Leader weight', 'Brokers', broker_ids, list(cluster_topologies.keys()), broker_leader_weights, ) for name, blc, blw in zip(list(cluster_topologies.keys()), broker_leader_counts, broker_leader_weights): print('\n' '{name}' 'Leader count imbalance: {net_imbalance}\n' 'Broker leader weight mean: {weight_mean}\n' 'Broker leader weight stdev: {weight_stdev}\n' 'Broker leader weight cv: {weight_cv}'.format( name='' if len(cluster_topologies) == 1 else name + '\n', net_imbalance=stats.get_net_imbalance(blc), weight_mean=stats.mean(blw), weight_stdev=stats.standard_deviation(blw), weight_cv=stats.coefficient_of_variation(blw), ))
def test_rebalance_leaders_unbalanced_case4( self, create_balancer, create_cluster_topology, ): # Imbalanced assignment # Partial leader-imbalance possible # (Broker: leader-count): {0: 3, 1: 1, 2:0} # opt-count: 5/3 = 1, extra-count = 2 assignment = dict( [ ((u'T0', 0), ['1', '2']), ((u'T0', 1), ['0', '2']), ((u'T1', 0), ['0']), ((u'T1', 1), ['0']), ((u'T1', 2), ['0']), ] ) ct = create_cluster_topology(assignment, broker_range(3)) net_imbal = get_net_imbalance( get_broker_leader_counts(list(ct.brokers.values())), ) cb = create_balancer(ct) cb.rebalance_leaders() new_leaders_per_broker = { broker.id: broker.count_preferred_replica() for broker in six.itervalues(ct.brokers) } new_net_imbal = get_net_imbalance(list(new_leaders_per_broker.values())) # Verify that net-imbalance has reduced but not zero assert new_net_imbal > 0 and new_net_imbal < net_imbal # Verify the changes in leaders-per-broker count assert new_leaders_per_broker['2'] == 1 assert new_leaders_per_broker['1'] == 1 assert new_leaders_per_broker['0'] == 3
def test_rebalance_leaders_unbalanced_case4( self, create_balancer, create_cluster_topology, ): # Imbalanced assignment # Partial leader-imbalance possible # (Broker: leader-count): {0: 3, 1: 1, 2:0} # opt-count: 5/3 = 1, extra-count = 2 assignment = dict( [ ((u'T0', 0), ['1', '2']), ((u'T0', 1), ['0', '2']), ((u'T1', 0), ['0']), ((u'T1', 1), ['0']), ((u'T1', 2), ['0']), ] ) ct = create_cluster_topology(assignment, broker_range(3)) net_imbal = get_net_imbalance( get_broker_leader_counts(ct.brokers.values()), ) cb = create_balancer(ct) cb.rebalance_leaders() new_leaders_per_broker = { broker.id: broker.count_preferred_replica() for broker in ct.brokers.itervalues() } new_net_imbal = get_net_imbalance(new_leaders_per_broker.values()) # Verify that net-imbalance has reduced but not zero assert new_net_imbal > 0 and new_net_imbal < net_imbal # Verify the changes in leaders-per-broker count assert new_leaders_per_broker['2'] == 1 assert new_leaders_per_broker['1'] == 1 assert new_leaders_per_broker['0'] == 3
def test_rebalance_leaders_unbalanced_case2b( self, create_balancer, create_cluster_topology, ): assignment = dict([ ((u'T0', 0), ['3', '2']), ((u'T1', 0), ['1', '2']), ((u'T1', 1), ['0', '1']), ((u'T2', 0), ['0']), ]) ct = create_cluster_topology(assignment, broker_range(4)) cb = create_balancer(ct) cb.rebalance_leaders() # Verify balanced leader_imbal = get_net_imbalance( get_broker_leader_counts(list(ct.brokers.values())), ) assert leader_imbal == 0
def test_revoke_leadership_multiple_brokers( self, create_balancer, create_cluster_topology, ): assignment = dict( [ ((u'T0', 0), ['2', '0']), ((u'T1', 0), ['2', '1']), ((u'T1', 1), ['0', '2']), ((u'T2', 0), ['0', '3']), ((u'T3', 0), ['3', '1']), ((u'T3', 1), ['3', '1']), ] ) ct = create_cluster_topology(assignment, broker_range(4)) cb = create_balancer(ct) cb.revoke_leadership(['2', '3']) new_leaders_per_broker = { broker.id: broker.count_preferred_replica() for broker in six.itervalues(ct.brokers) } _, total_movements = \ calculate_partition_movement(assignment, ct.assignment) # Get net imbalance statistics excluding brokers to be revoked # leadership from brokers = [ b for b in six.itervalues(ct.brokers) if b.id not in ['2', '3'] ] new_net_imbal = get_net_imbalance(get_broker_leader_counts(brokers)) # Verify that broker '2' and '3' is not leader of any partition assert new_leaders_per_broker['0'] == 3 assert new_leaders_per_broker['1'] == 3 assert new_leaders_per_broker['2'] == 0 assert new_leaders_per_broker['3'] == 0 # Assert no partition movements assert total_movements == 0 # Assert remaining brokers are balanced with leader count assert new_net_imbal == 0
def test_rebalance_leaders_unbalanced_case2b( self, create_balancer, create_cluster_topology, ): assignment = dict( [ ((u'T0', 0), ['3', '2']), ((u'T1', 0), ['1', '2']), ((u'T1', 1), ['0', '1']), ((u'T2', 0), ['0']), ] ) ct = create_cluster_topology(assignment, broker_range(4)) cb = create_balancer(ct) cb.rebalance_leaders() # Verify balanced leader_imbal = get_net_imbalance( get_broker_leader_counts(list(ct.brokers.values())), ) assert leader_imbal == 0
def test_rebalance_leaders_unbalanced_case1( self, create_balancer, create_cluster_topology, ): # Balance leader-imbalance successfully # (broker-id: leader-count): {0: 0, 1:2, 2:1} # Net-leader-imbalance: 1 # opt-count: 3/3 = 1, extra-count: 3%3 = 0 assignment = dict( [ ((u'T0', 0), ['1', '2']), ((u'T0', 1), ['2', '0']), ((u'T1', 0), ['1', '0']), ] ) ct = create_cluster_topology(assignment, broker_range(3)) orig_assignment = ct.assignment cb = create_balancer(ct) cb.rebalance_leaders() # Verify if valid-leader assignment self.assert_leader_valid(orig_assignment, ct.assignment) # New-leader imbalance-count be less than previous imbal count new_leaders_per_broker = { broker.id: broker.count_preferred_replica() for broker in six.itervalues(ct.brokers) } new_leader_imbal = get_net_imbalance(list(new_leaders_per_broker.values())) # Verify leader-balanced assert new_leader_imbal == 0 # Verify partitions-changed assignment assert new_leaders_per_broker['0'] == 1 assert new_leaders_per_broker['1'] == 1 assert new_leaders_per_broker['2'] == 1
def display_leader_imbalance(cluster_topologies): """Display leader count and weight imbalance statistics. :param cluster_topologies: A dictionary mapping a string name to a ClusterTopology object. """ broker_ids = list(next(six.itervalues(cluster_topologies)).brokers.keys()) assert all( set(broker_ids) == set(cluster_topology.brokers.keys()) for cluster_topology in six.itervalues(cluster_topologies) ) broker_leader_counts = [ stats.get_broker_leader_counts( cluster_topology.brokers[broker_id] for broker_id in broker_ids ) for cluster_topology in six.itervalues(cluster_topologies) ] broker_leader_weights = [ stats.get_broker_leader_weights( cluster_topology.brokers[broker_id] for broker_id in broker_ids ) for cluster_topology in six.itervalues(cluster_topologies) ] _display_table_title_multicolumn( 'Leader Count', 'Brokers', broker_ids, list(cluster_topologies.keys()), broker_leader_counts, ) print('') _display_table_title_multicolumn( 'Leader weight', 'Brokers', broker_ids, list(cluster_topologies.keys()), broker_leader_weights, ) for name, blc, blw in zip( list(cluster_topologies.keys()), broker_leader_counts, broker_leader_weights ): print( '\n' '{name}' 'Leader count imbalance: {net_imbalance}\n' 'Broker leader weight mean: {weight_mean}\n' 'Broker leader weight stdev: {weight_stdev}\n' 'Broker leader weight cv: {weight_cv}' .format( name='' if len(cluster_topologies) == 1 else name + '\n', net_imbalance=stats.get_net_imbalance(blc), weight_mean=stats.mean(blw), weight_stdev=stats.stdevp(blw), weight_cv=stats.coefficient_of_variation(blw), ) )
def test_get_net_imbalance_imbalanced_equal(): assert stats.get_net_imbalance([3, 2, 3, 4, 3]) == 1
def test_get_net_imbalance_imbalanced_unequal(): assert stats.get_net_imbalance([3, 2, 4, 4, 4]) == 1
def test_get_net_imbalance_imbalanced_unequal(): assert stats.get_net_imbalance([3, 2, 4, 4, 4]) == 1
def test_get_net_imbalance_imbalanced_equal(): assert stats.get_net_imbalance([3, 2, 3, 4, 3]) == 1