def test_rebalance_brokers_for_topic_partition_imbalance(self, create_partition): # Broker Topics:Partition # 1 All partitions: 4 from topic1, 2 from topic2 and 1 each from topic3 and topic4 # 2 Newly added broker:(empty) # total partitions: 8 p10 = create_partition('topic1', 0) p11 = create_partition('topic1', 1) p12 = create_partition('topic1', 2) p13 = create_partition('topic1', 3) p20 = create_partition('topic2', 0) p21 = create_partition('topic2', 1) p30 = create_partition('topic3', 0) p40 = create_partition('topic4', 0) b1 = create_broker('b1', [p10, p11, p12, p13, p20, p21, p30, p40]) b2 = create_broker('b2', []) rg = ReplicationGroup('test_rg', set([b1, b2])) rg.rebalance_brokers() possible_topics1 = set([p10.topic, p11.topic, p20.topic, p30.topic]) possible_topics2 = set([p10.topic, p11.topic, p20.topic, p40.topic]) assert ( b1.topics == possible_topics1 and b2.topics == possible_topics2 ) or ( b1.topics == possible_topics2 and b2.topics == possible_topics1 )
def test_decommission_not_enough_replicas(self, create_partition): p10 = create_partition('topic1', 0) p11 = create_partition('topic1', 1) p20 = create_partition('topic2', 0, replication_factor=2) b1 = create_broker('b1', [p10, p11, p20]) b2 = create_broker('b1', [p20]) rg = ReplicationGroup('rg', set([b1, b2])) b2.mark_decommissioned() rg.rebalance_brokers() assert not b2.empty()
def test_decommission_no_remaining_brokers(self, create_partition): p10 = create_partition('topic1', 0) p11 = create_partition('topic1', 1) p20 = create_partition('topic2', 0) b1 = create_broker('b1', [p10, p11, p20]) b2 = create_broker('b2', []) b2.mark_inactive() rg = ReplicationGroup('rg', set([b1, b2])) b1.mark_decommissioned() # Two brokers b1 decommissioned b2 inactive with pytest.raises(EmptyReplicationGroupError): rg.rebalance_brokers()
def test_decommission_multi_replicas_edge_case(self, create_partition): # After decommissioning there will be 2 brokers with 5 partitions and 1 # with 4. Because of replication constraints (2 replicas of the same # partition can't live in the same broker) the algorithm has to be smart # enough to find a solution. p10 = create_partition('topic1', 0) p11 = create_partition('topic1', 1) p20 = create_partition('topic2', 0, replication_factor=3) p21 = create_partition('topic2', 1, replication_factor=3) p22 = create_partition('topic2', 2, replication_factor=3) p23 = create_partition('topic2', 3, replication_factor=3) b1 = create_broker('b1', [p10, p20, p23]) b2 = create_broker('b2', [p11, p21, p22, p23]) b3 = create_broker('b3', [p20, p21, p22, p23]) b4 = create_broker('b4', [p20, p21, p22]) rg = ReplicationGroup('rg', set([b1, b2, b3, b4])) b1.mark_decommissioned() rg.rebalance_brokers() assert b1.empty()
def test_rebalance_brokers_for_topic_partition_imbalance( self, create_partition): # Broker Topics:Partition # 1 All partitions: 4 from topic1, 2 from topic2 and 1 each from topic3 and topic4 # 2 Newly added broker:(empty) # total partitions: 8 p10 = create_partition('topic1', 0) p11 = create_partition('topic1', 1) p12 = create_partition('topic1', 2) p13 = create_partition('topic1', 3) p20 = create_partition('topic2', 0) p21 = create_partition('topic2', 1) p30 = create_partition('topic3', 0) p40 = create_partition('topic4', 0) b1 = create_broker('b1', [p10, p11, p12, p13, p20, p21, p30, p40]) b2 = create_broker('b2', []) rg = ReplicationGroup('test_rg', set([b1, b2])) rg.rebalance_brokers() possible_topics1 = set([p10.topic, p11.topic, p20.topic, p30.topic]) possible_topics2 = set([p10.topic, p11.topic, p20.topic, p40.topic]) assert (b1.topics == possible_topics1 and b2.topics == possible_topics2) or (b1.topics == possible_topics2 and b2.topics == possible_topics1)
def test_rebalance_empty_replication_group(self): rg = ReplicationGroup('empty_rg') with pytest.raises(EmptyReplicationGroupError): rg.rebalance_brokers()