def test_process_cluster_empty_broker(self):
self.cluster.add_broker(Broker('brokerhost3.example.com', id=3))
b1 = self.cluster.brokers[1]
b2 = self.cluster.brokers[2]
self.cluster.add_topic(Topic("testTopic3", 2))
partition = self.cluster.topics['testTopic3'].partitions[0]
partition.size = 1000
partition.add_replica(b1, 0)
partition.add_replica(b2, 1)
partition = self.cluster.topics['testTopic3'].partitions[1]
partition.add_replica(b2, 0)
partition.add_replica(b1, 1)
partition.size = 2000
action = ActionBalanceSize(self.args, self.cluster)
action.process_cluster()
assert sum([p.size for p in self.cluster.brokers[1].partitions[0]],
0) == 3000
assert sum([p.size for p in self.cluster.brokers[1].partitions[1]],
0) == 3000
assert sum([p.size for p in self.cluster.brokers[2].partitions[0]],
0) == 3000
assert sum([p.size for p in self.cluster.brokers[2].partitions[1]],
0) == 3000
assert sum([p.size for p in self.cluster.brokers[3].partitions[0]],
0) == 3000
assert sum([p.size for p in self.cluster.brokers[3].partitions[1]],
0) == 3000
def test_process_cluster_large_partition_early(self):
b1 = self.cluster.brokers[1]
b2 = self.cluster.brokers[2]
self.cluster.add_topic(Topic("testTopic3", 3))
partition = self.cluster.topics['testTopic3'].partitions[0]
partition.size = 1000
partition.add_replica(b1, 0)
partition.add_replica(b2, 1)
partition = self.cluster.topics['testTopic3'].partitions[1]
partition.add_replica(b1, 0)
partition.add_replica(b2, 1)
partition.size = 2000
partition = self.cluster.topics['testTopic3'].partitions[2]
partition.add_replica(b1, 0)
partition.add_replica(b2, 1)
partition.size = 1000
self.cluster.topics['testTopic1'].partitions[0].size = 8000
action = ActionBalanceSize(self.args, self.cluster)
action.process_cluster()
b1_0 = sum([p.size for p in self.cluster.brokers[1].partitions[0]], 0)
b1_1 = sum([p.size for p in self.cluster.brokers[1].partitions[1]], 0)
b2_0 = sum([p.size for p in self.cluster.brokers[2].partitions[0]], 0)
b2_1 = sum([p.size for p in self.cluster.brokers[2].partitions[1]], 0)
assert b1_0 >= 8000 and b1_0 <= 9000
assert b1_1 >= 8000 and b1_1 <= 9000
assert b2_0 >= 8000 and b2_0 <= 9000
assert b2_1 >= 8000 and b2_1 <= 9000
def test_process_cluster_large_partition_early(self):
b1 = self.cluster.brokers[1]
b2 = self.cluster.brokers[2]
self.cluster.add_topic(Topic("testTopic3", 3))
partition = self.cluster.topics['testTopic3'].partitions[0]
partition.size = 1000
partition.add_replica(b1, 0)
partition.add_replica(b2, 1)
partition = self.cluster.topics['testTopic3'].partitions[1]
partition.add_replica(b1, 0)
partition.add_replica(b2, 1)
partition.size = 2000
partition = self.cluster.topics['testTopic3'].partitions[2]
partition.add_replica(b1, 0)
partition.add_replica(b2, 1)
partition.size = 1000
self.cluster.topics['testTopic1'].partitions[0].size = 8000
action = ActionBalanceSize(self.args, self.cluster)
action.process_cluster()
b1_0 = sum([p.size for p in self.cluster.brokers[1].partitions[0]], 0)
b1_1 = sum([p.size for p in self.cluster.brokers[1].partitions[1]], 0)
b2_0 = sum([p.size for p in self.cluster.brokers[2].partitions[0]], 0)
b2_1 = sum([p.size for p in self.cluster.brokers[2].partitions[1]], 0)
assert b1_0 >= 8000 and b1_0 <= 9000
assert b1_1 >= 8000 and b1_1 <= 9000
assert b2_0 >= 8000 and b2_0 <= 9000
assert b2_1 >= 8000 and b2_1 <= 9000
def test_process_cluster_odd_partitions(self):
b1 = self.cluster.brokers[1]
b2 = self.cluster.brokers[2]
self.cluster.add_topic(Topic("testTopic3", 3))
partition = self.cluster.topics['testTopic3'].partitions[0]
partition.size = 1000
partition.add_replica(b1, 0)
partition.add_replica(b2, 1)
partition = self.cluster.topics['testTopic3'].partitions[1]
partition.add_replica(b2, 0)
partition.add_replica(b1, 1)
partition.size = 2000
partition = self.cluster.topics['testTopic3'].partitions[2]
partition.add_replica(b2, 0)
partition.add_replica(b1, 1)
partition.size = 1000
action = ActionBalanceSize(self.args, self.cluster)
action.process_cluster()
assert sum([p.size for p in self.cluster.brokers[1].partitions[0]],
0) == 5000
assert sum([p.size for p in self.cluster.brokers[1].partitions[1]],
0) == 5000
assert sum([p.size for p in self.cluster.brokers[2].partitions[0]],
0) == 5000
assert sum([p.size for p in self.cluster.brokers[2].partitions[1]],
0) == 5000
def test_process_cluster_no_change(self):
action = ActionBalanceSize(self.args, self.cluster)
action.process_cluster()
b1 = self.cluster.brokers[1]
b2 = self.cluster.brokers[2]
assert self.cluster.topics['testTopic1'].partitions[0].replicas == [b1, b2]
assert self.cluster.topics['testTopic1'].partitions[1].replicas == [b2, b1]
assert self.cluster.topics['testTopic2'].partitions[0].replicas == [b2, b1]
assert self.cluster.topics['testTopic2'].partitions[1].replicas == [b1, b2]
def test_process_cluster_one_move(self):
b1 = self.cluster.brokers[1]
b2 = self.cluster.brokers[2]
self.cluster.topics['testTopic1'].partitions[0].swap_replica_positions(b1, b2)
action = ActionBalanceSize(self.args, self.cluster)
action.process_cluster()
assert sum([p.size for p in self.cluster.brokers[1].partitions[0]], 0) == 3000
assert sum([p.size for p in self.cluster.brokers[1].partitions[1]], 0) == 3000
assert sum([p.size for p in self.cluster.brokers[2].partitions[0]], 0) == 3000
assert sum([p.size for p in self.cluster.brokers[2].partitions[1]], 0) == 3000
class ActionBalanceRate(ActionBalanceModule):
name = "rate"
helpstr = "Move the busiest partitions in the cluster to even the total bytes rate per-broker for each replica position"
def __init__(self, args, cluster):
super(ActionBalanceRate, self).__init__(args, cluster)
# This module merely calls the balance "size" module with the scaled_size attr, so we're going to instantiate a copy of that
self._size = ActionBalanceSize(args, cluster, size_attr='scaled_size')
def process_cluster(self):
# Call the size module
self._size.process_cluster()
def test_process_cluster_one_move(self):
b1 = self.cluster.brokers[1]
b2 = self.cluster.brokers[2]
self.cluster.topics['testTopic1'].partitions[0].swap_replica_positions(
b1, b2)
action = ActionBalanceSize(self.args, self.cluster)
action.process_cluster()
assert sum([p.size for p in self.cluster.brokers[1].partitions[0]],
0) == 3000
assert sum([p.size for p in self.cluster.brokers[1].partitions[1]],
0) == 3000
assert sum([p.size for p in self.cluster.brokers[2].partitions[0]],
0) == 3000
assert sum([p.size for p in self.cluster.brokers[2].partitions[1]],
0) == 3000
def test_process_cluster_no_change(self):
action = ActionBalanceSize(self.args, self.cluster)
action.process_cluster()
b1 = self.cluster.brokers[1]
b2 = self.cluster.brokers[2]
assert self.cluster.topics['testTopic1'].partitions[0].replicas == [
b1, b2
]
assert self.cluster.topics['testTopic1'].partitions[1].replicas == [
b2, b1
]
assert self.cluster.topics['testTopic2'].partitions[0].replicas == [
b2, b1
]
assert self.cluster.topics['testTopic2'].partitions[1].replicas == [
b1, b2
]
def test_process_cluster_empty_broker(self):
self.cluster.add_broker(Broker(3, 'brokerhost3.example.com'))
b1 = self.cluster.brokers[1]
b2 = self.cluster.brokers[2]
self.cluster.add_topic(Topic("testTopic3", 2))
partition = self.cluster.topics['testTopic3'].partitions[0]
partition.size = 1000
partition.add_replica(b1, 0)
partition.add_replica(b2, 1)
partition = self.cluster.topics['testTopic3'].partitions[1]
partition.add_replica(b2, 0)
partition.add_replica(b1, 1)
partition.size = 2000
action = ActionBalanceSize(self.args, self.cluster)
action.process_cluster()
assert sum([p.size for p in self.cluster.brokers[1].partitions[0]], 0) == 3000
assert sum([p.size for p in self.cluster.brokers[1].partitions[1]], 0) == 3000
assert sum([p.size for p in self.cluster.brokers[2].partitions[0]], 0) == 3000
assert sum([p.size for p in self.cluster.brokers[2].partitions[1]], 0) == 3000
assert sum([p.size for p in self.cluster.brokers[3].partitions[0]], 0) == 3000
assert sum([p.size for p in self.cluster.brokers[3].partitions[1]], 0) == 3000
def test_process_cluster_odd_partitions(self):
b1 = self.cluster.brokers[1]
b2 = self.cluster.brokers[2]
self.cluster.add_topic(Topic("testTopic3", 3))
partition = self.cluster.topics['testTopic3'].partitions[0]
partition.size = 1000
partition.add_replica(b1, 0)
partition.add_replica(b2, 1)
partition = self.cluster.topics['testTopic3'].partitions[1]
partition.add_replica(b2, 0)
partition.add_replica(b1, 1)
partition.size = 2000
partition = self.cluster.topics['testTopic3'].partitions[2]
partition.add_replica(b2, 0)
partition.add_replica(b1, 1)
partition.size = 1000
action = ActionBalanceSize(self.args, self.cluster)
action.process_cluster()
assert sum([p.size for p in self.cluster.brokers[1].partitions[0]], 0) == 5000
assert sum([p.size for p in self.cluster.brokers[1].partitions[1]], 0) == 5000
assert sum([p.size for p in self.cluster.brokers[2].partitions[0]], 0) == 5000
assert sum([p.size for p in self.cluster.brokers[2].partitions[1]], 0) == 5000
def test_create_class(self):
action = ActionBalanceSize(self.args, self.cluster)
assert isinstance(action, ActionBalanceSize)
def __init__(self, args, cluster):
super(ActionBalanceRate, self).__init__(args, cluster)
# This module merely calls the balance "size" module with the scaled_size attr, so we're going to instantiate a copy of that
self._size = ActionBalanceSize(args, cluster, size_attr='scaled_size')