def test_process_cluster_empty_broker(self): self.cluster.add_broker(Broker('brokerhost3.example.com', id=3)) b1 = self.cluster.brokers[1] b2 = self.cluster.brokers[2] self.cluster.add_topic(Topic("testTopic3", 2)) partition = self.cluster.topics['testTopic3'].partitions[0] partition.size = 1000 partition.add_replica(b1, 0) partition.add_replica(b2, 1) partition = self.cluster.topics['testTopic3'].partitions[1] partition.add_replica(b2, 0) partition.add_replica(b1, 1) partition.size = 2000 action = ActionBalanceSize(self.args, self.cluster) action.process_cluster() assert sum([p.size for p in self.cluster.brokers[1].partitions[0]], 0) == 3000 assert sum([p.size for p in self.cluster.brokers[1].partitions[1]], 0) == 3000 assert sum([p.size for p in self.cluster.brokers[2].partitions[0]], 0) == 3000 assert sum([p.size for p in self.cluster.brokers[2].partitions[1]], 0) == 3000 assert sum([p.size for p in self.cluster.brokers[3].partitions[0]], 0) == 3000 assert sum([p.size for p in self.cluster.brokers[3].partitions[1]], 0) == 3000
def test_process_cluster_large_partition_early(self): b1 = self.cluster.brokers[1] b2 = self.cluster.brokers[2] self.cluster.add_topic(Topic("testTopic3", 3)) partition = self.cluster.topics['testTopic3'].partitions[0] partition.size = 1000 partition.add_replica(b1, 0) partition.add_replica(b2, 1) partition = self.cluster.topics['testTopic3'].partitions[1] partition.add_replica(b1, 0) partition.add_replica(b2, 1) partition.size = 2000 partition = self.cluster.topics['testTopic3'].partitions[2] partition.add_replica(b1, 0) partition.add_replica(b2, 1) partition.size = 1000 self.cluster.topics['testTopic1'].partitions[0].size = 8000 action = ActionBalanceSize(self.args, self.cluster) action.process_cluster() b1_0 = sum([p.size for p in self.cluster.brokers[1].partitions[0]], 0) b1_1 = sum([p.size for p in self.cluster.brokers[1].partitions[1]], 0) b2_0 = sum([p.size for p in self.cluster.brokers[2].partitions[0]], 0) b2_1 = sum([p.size for p in self.cluster.brokers[2].partitions[1]], 0) assert b1_0 >= 8000 and b1_0 <= 9000 assert b1_1 >= 8000 and b1_1 <= 9000 assert b2_0 >= 8000 and b2_0 <= 9000 assert b2_1 >= 8000 and b2_1 <= 9000
def test_process_cluster_large_partition_early(self): b1 = self.cluster.brokers[1] b2 = self.cluster.brokers[2] self.cluster.add_topic(Topic("testTopic3", 3)) partition = self.cluster.topics['testTopic3'].partitions[0] partition.size = 1000 partition.add_replica(b1, 0) partition.add_replica(b2, 1) partition = self.cluster.topics['testTopic3'].partitions[1] partition.add_replica(b1, 0) partition.add_replica(b2, 1) partition.size = 2000 partition = self.cluster.topics['testTopic3'].partitions[2] partition.add_replica(b1, 0) partition.add_replica(b2, 1) partition.size = 1000 self.cluster.topics['testTopic1'].partitions[0].size = 8000 action = ActionBalanceSize(self.args, self.cluster) action.process_cluster() b1_0 = sum([p.size for p in self.cluster.brokers[1].partitions[0]], 0) b1_1 = sum([p.size for p in self.cluster.brokers[1].partitions[1]], 0) b2_0 = sum([p.size for p in self.cluster.brokers[2].partitions[0]], 0) b2_1 = sum([p.size for p in self.cluster.brokers[2].partitions[1]], 0) assert b1_0 >= 8000 and b1_0 <= 9000 assert b1_1 >= 8000 and b1_1 <= 9000 assert b2_0 >= 8000 and b2_0 <= 9000 assert b2_1 >= 8000 and b2_1 <= 9000
def test_process_cluster_odd_partitions(self): b1 = self.cluster.brokers[1] b2 = self.cluster.brokers[2] self.cluster.add_topic(Topic("testTopic3", 3)) partition = self.cluster.topics['testTopic3'].partitions[0] partition.size = 1000 partition.add_replica(b1, 0) partition.add_replica(b2, 1) partition = self.cluster.topics['testTopic3'].partitions[1] partition.add_replica(b2, 0) partition.add_replica(b1, 1) partition.size = 2000 partition = self.cluster.topics['testTopic3'].partitions[2] partition.add_replica(b2, 0) partition.add_replica(b1, 1) partition.size = 1000 action = ActionBalanceSize(self.args, self.cluster) action.process_cluster() assert sum([p.size for p in self.cluster.brokers[1].partitions[0]], 0) == 5000 assert sum([p.size for p in self.cluster.brokers[1].partitions[1]], 0) == 5000 assert sum([p.size for p in self.cluster.brokers[2].partitions[0]], 0) == 5000 assert sum([p.size for p in self.cluster.brokers[2].partitions[1]], 0) == 5000
def test_process_cluster_no_change(self): action = ActionBalanceSize(self.args, self.cluster) action.process_cluster() b1 = self.cluster.brokers[1] b2 = self.cluster.brokers[2] assert self.cluster.topics['testTopic1'].partitions[0].replicas == [b1, b2] assert self.cluster.topics['testTopic1'].partitions[1].replicas == [b2, b1] assert self.cluster.topics['testTopic2'].partitions[0].replicas == [b2, b1] assert self.cluster.topics['testTopic2'].partitions[1].replicas == [b1, b2]
def test_process_cluster_one_move(self): b1 = self.cluster.brokers[1] b2 = self.cluster.brokers[2] self.cluster.topics['testTopic1'].partitions[0].swap_replica_positions(b1, b2) action = ActionBalanceSize(self.args, self.cluster) action.process_cluster() assert sum([p.size for p in self.cluster.brokers[1].partitions[0]], 0) == 3000 assert sum([p.size for p in self.cluster.brokers[1].partitions[1]], 0) == 3000 assert sum([p.size for p in self.cluster.brokers[2].partitions[0]], 0) == 3000 assert sum([p.size for p in self.cluster.brokers[2].partitions[1]], 0) == 3000
class ActionBalanceRate(ActionBalanceModule): name = "rate" helpstr = "Move the busiest partitions in the cluster to even the total bytes rate per-broker for each replica position" def __init__(self, args, cluster): super(ActionBalanceRate, self).__init__(args, cluster) # This module merely calls the balance "size" module with the scaled_size attr, so we're going to instantiate a copy of that self._size = ActionBalanceSize(args, cluster, size_attr='scaled_size') def process_cluster(self): # Call the size module self._size.process_cluster()
def test_process_cluster_one_move(self): b1 = self.cluster.brokers[1] b2 = self.cluster.brokers[2] self.cluster.topics['testTopic1'].partitions[0].swap_replica_positions( b1, b2) action = ActionBalanceSize(self.args, self.cluster) action.process_cluster() assert sum([p.size for p in self.cluster.brokers[1].partitions[0]], 0) == 3000 assert sum([p.size for p in self.cluster.brokers[1].partitions[1]], 0) == 3000 assert sum([p.size for p in self.cluster.brokers[2].partitions[0]], 0) == 3000 assert sum([p.size for p in self.cluster.brokers[2].partitions[1]], 0) == 3000
def test_process_cluster_no_change(self): action = ActionBalanceSize(self.args, self.cluster) action.process_cluster() b1 = self.cluster.brokers[1] b2 = self.cluster.brokers[2] assert self.cluster.topics['testTopic1'].partitions[0].replicas == [ b1, b2 ] assert self.cluster.topics['testTopic1'].partitions[1].replicas == [ b2, b1 ] assert self.cluster.topics['testTopic2'].partitions[0].replicas == [ b2, b1 ] assert self.cluster.topics['testTopic2'].partitions[1].replicas == [ b1, b2 ]
def test_process_cluster_empty_broker(self): self.cluster.add_broker(Broker(3, 'brokerhost3.example.com')) b1 = self.cluster.brokers[1] b2 = self.cluster.brokers[2] self.cluster.add_topic(Topic("testTopic3", 2)) partition = self.cluster.topics['testTopic3'].partitions[0] partition.size = 1000 partition.add_replica(b1, 0) partition.add_replica(b2, 1) partition = self.cluster.topics['testTopic3'].partitions[1] partition.add_replica(b2, 0) partition.add_replica(b1, 1) partition.size = 2000 action = ActionBalanceSize(self.args, self.cluster) action.process_cluster() assert sum([p.size for p in self.cluster.brokers[1].partitions[0]], 0) == 3000 assert sum([p.size for p in self.cluster.brokers[1].partitions[1]], 0) == 3000 assert sum([p.size for p in self.cluster.brokers[2].partitions[0]], 0) == 3000 assert sum([p.size for p in self.cluster.brokers[2].partitions[1]], 0) == 3000 assert sum([p.size for p in self.cluster.brokers[3].partitions[0]], 0) == 3000 assert sum([p.size for p in self.cluster.brokers[3].partitions[1]], 0) == 3000
def test_process_cluster_odd_partitions(self): b1 = self.cluster.brokers[1] b2 = self.cluster.brokers[2] self.cluster.add_topic(Topic("testTopic3", 3)) partition = self.cluster.topics['testTopic3'].partitions[0] partition.size = 1000 partition.add_replica(b1, 0) partition.add_replica(b2, 1) partition = self.cluster.topics['testTopic3'].partitions[1] partition.add_replica(b2, 0) partition.add_replica(b1, 1) partition.size = 2000 partition = self.cluster.topics['testTopic3'].partitions[2] partition.add_replica(b2, 0) partition.add_replica(b1, 1) partition.size = 1000 action = ActionBalanceSize(self.args, self.cluster) action.process_cluster() assert sum([p.size for p in self.cluster.brokers[1].partitions[0]], 0) == 5000 assert sum([p.size for p in self.cluster.brokers[1].partitions[1]], 0) == 5000 assert sum([p.size for p in self.cluster.brokers[2].partitions[0]], 0) == 5000 assert sum([p.size for p in self.cluster.brokers[2].partitions[1]], 0) == 5000