def test_cluster_create_from_zookeeper(self):
self.mock_children.side_effect = [['1', '2'],
['testTopic1', 'testTopic2']]
self.mock_get.side_effect = [
(('{"jmx_port":7667,"timestamp":"1465289114807","endpoints":["PLAINTEXT://brokerhost1.example.com:9223",'
'"SSL://brokerhost1.example.com:9224"],"host":"brokerhost1.example.com","version":1,"port":9223}'
), None),
(('{"jmx_port":7667,"timestamp":"1465289114807","endpoints":["PLAINTEXT://brokerhost2.example.com:9223",'
'"SSL://brokerhost2.example.com:9224"],"host":"brokerhost2.example.com","version":1,"port":9223}'
), None),
('{"version":1,"partitions":{"0":[1,2],"1":[2,1]}}', None),
('{"version":1,"partitions":{"0":[2,1],"1":[1,2]}}', None)
]
cluster = Cluster.create_from_zookeeper('zkconnect')
assert len(cluster.brokers) == 2
b1 = cluster.brokers[1]
b2 = cluster.brokers[2]
assert b1.hostname == 'brokerhost1.example.com'
assert b2.hostname == 'brokerhost2.example.com'
assert len(cluster.topics) == 2
assert 'testTopic1' in cluster.topics
assert 'testTopic2' in cluster.topics
t1 = cluster.topics['testTopic1']
assert len(t1.partitions) == 2
assert t1.partitions[0].replicas == [b1, b2]
assert t1.partitions[1].replicas == [b2, b1]
t2 = cluster.topics['testTopic2']
assert len(t2.partitions) == 2
assert t2.partitions[0].replicas == [b2, b1]
assert t2.partitions[1].replicas == [b1, b2]
def test_cluster_create_from_zookeeper(self):
self.mock_children.side_effect = [['1', '2'], ['testTopic1', 'testTopic2']]
self.mock_get.side_effect = [(('{"jmx_port":7667,"timestamp":"1465289114807","endpoints":["PLAINTEXT://brokerhost1.example.com:9223",'
'"SSL://brokerhost1.example.com:9224"],"host":"brokerhost1.example.com","version":1,"port":9223}'), None),
(('{"jmx_port":7667,"timestamp":"1465289114807","endpoints":["PLAINTEXT://brokerhost2.example.com:9223",'
'"SSL://brokerhost2.example.com:9224"],"host":"brokerhost2.example.com","version":1,"port":9223}'), None),
('{"version":1,"partitions":{"0":[1,2],"1":[2,1]}}', None),
('{"version":1,"partitions":{"0":[2,1],"1":[1,2]}}', None)]
cluster = Cluster.create_from_zookeeper('zkconnect')
assert len(cluster.brokers) == 2
b1 = cluster.brokers[1]
b2 = cluster.brokers[2]
assert b1.hostname == 'brokerhost1.example.com'
assert b2.hostname == 'brokerhost2.example.com'
assert len(cluster.topics) == 2
assert 'testTopic1' in cluster.topics
assert 'testTopic2' in cluster.topics
t1 = cluster.topics['testTopic1']
assert len(t1.partitions) == 2
assert t1.partitions[0].replicas == [b1, b2]
assert t1.partitions[1].replicas == [b2, b1]
t2 = cluster.topics['testTopic2']
assert len(t2.partitions) == 2
assert t2.partitions[0].replicas == [b2, b1]
assert t2.partitions[1].replicas == [b1, b2]
def main():
# Start by loading all the modules
action_map = get_module_map(kafka.tools.assigner.actions, kafka.tools.assigner.actions.ActionModule)
sizer_map = get_module_map(kafka.tools.assigner.sizers, kafka.tools.assigner.sizers.SizerModule)
plugins = get_all_plugins()
# Set up and parse all CLI arguments
args = set_up_arguments(action_map, sizer_map, plugins)
run_plugins_at_step(plugins, 'set_arguments', args)
tools_path = get_tools_path(args.tools_path)
check_java_home()
cluster = Cluster.create_from_zookeeper(args.zookeeper)
run_plugins_at_step(plugins, 'set_cluster', cluster)
# If the module needs the partition sizes, call a size module to get the information
check_and_get_sizes(action_map[args.action], args, cluster, sizer_map)
run_plugins_at_step(plugins, 'after_sizes')
print_leadership("before", cluster, args.leadership)
# Clone the cluster, and run the action to generate a new cluster state
newcluster = cluster.clone()
action_to_run = action_map[args.action](args, newcluster)
action_to_run.process_cluster()
run_plugins_at_step(plugins, 'set_new_cluster', action_to_run.cluster)
print_leadership("after", newcluster, args.leadership)
move_partitions = cluster.changed_partitions(action_to_run.cluster)
batches = split_partitions_into_batches(move_partitions, batch_size=args.moves, use_class=Reassignment)
run_plugins_at_step(plugins, 'set_batches', batches)
log.info("Partition moves required: {0}".format(len(move_partitions)))
log.info("Number of batches: {0}".format(len(batches)))
dry_run = is_dry_run(args)
for i, batch in enumerate(batches):
log.info("Executing partition reassignment {0}/{1}: {2}".format(i + 1, len(batches), repr(batch)))
batch.execute(i + 1, len(batches), args.zookeeper, tools_path, plugins, dry_run)
run_plugins_at_step(plugins, 'before_ple')
if not args.skip_ple:
all_cluster_partitions = [p for p in action_to_run.cluster.partitions()]
batches = split_partitions_into_batches(all_cluster_partitions, batch_size=args.ple_size, use_class=ReplicaElection)
log.info("Number of replica elections: {0}".format(len(batches)))
run_preferred_replica_elections(batches, args, tools_path, plugins, dry_run)
run_plugins_at_step(plugins, 'finished')
return os.EX_OK
def test_cluster_create_missing_broker(self):
self.mock_children.side_effect = [['1', '2'], ['testTopic1', 'testTopic2']]
self.mock_get.side_effect = [(('{"jmx_port":7667,"timestamp":"1465289114807","endpoints":["PLAINTEXT://brokerhost1.example.com:9223",'
'"SSL://brokerhost1.example.com:9224"],"host":"brokerhost1.example.com","version":1,"port":9223}'), None),
(('{"jmx_port":7667,"timestamp":"1465289114807","endpoints":["PLAINTEXT://brokerhost2.example.com:9223",'
'"SSL://brokerhost2.example.com:9224"],"host":"brokerhost2.example.com","version":1,"port":9223}'), None),
('{"version":1,"partitions":{"0":[1,2],"1":[3,1]}}', None),
('{"version":1,"partitions":{"0":[3,1],"1":[1,2]}}', None)]
cluster = Cluster.create_from_zookeeper('zkconnect')
assert len(cluster.brokers) == 3
b1 = cluster.brokers[1]
b3 = cluster.brokers[3]
assert cluster.brokers[3].hostname is None
assert cluster.topics['testTopic1'].partitions[1].replicas == [b3, b1]
assert cluster.topics['testTopic2'].partitions[0].replicas == [b3, b1]
def test_cluster_create_with_retention(self):
self.mock_children.side_effect = [['1', '2'],
['testTopic1', 'testTopic2']]
self.mock_get.side_effect = [
(('{"jmx_port":7667,"timestamp":"1465289114807","endpoints":["PLAINTEXT://brokerhost1.example.com:9223",'
'"SSL://brokerhost1.example.com:9224"],"host":"brokerhost1.example.com","version":1,"port":9223}'
), None),
(('{"jmx_port":7667,"timestamp":"1465289114807","endpoints":["PLAINTEXT://brokerhost2.example.com:9223",'
'"SSL://brokerhost2.example.com:9224"],"host":"brokerhost2.example.com","version":1,"port":9223}'
), None),
('{"version":1,"partitions":{"0":[1,2],"1":[2,1]}}', None),
('{"version":1,"config":{}}', None),
('{"version":1,"partitions":{"0":[2,1],"1":[1,2]}}', None),
('{"version":1,"config":{"retention.ms":"172800000"}}', None)
]
cluster = Cluster.create_from_zookeeper('zkconnect')
assert cluster.topics['testTopic1'].retention == 1
assert cluster.topics['testTopic2'].retention == 172800000
def test_cluster_create_missing_broker(self):
self.mock_children.side_effect = [['1', '2'],
['testTopic1', 'testTopic2']]
self.mock_get.side_effect = [
(('{"jmx_port":7667,"timestamp":"1465289114807","endpoints":["PLAINTEXT://brokerhost1.example.com:9223",'
'"SSL://brokerhost1.example.com:9224"],"host":"brokerhost1.example.com","version":1,"port":9223}'
), None),
(('{"jmx_port":7667,"timestamp":"1465289114807","endpoints":["PLAINTEXT://brokerhost2.example.com:9223",'
'"SSL://brokerhost2.example.com:9224"],"host":"brokerhost2.example.com","version":1,"port":9223}'
), None),
('{"version":1,"partitions":{"0":[1,2],"1":[3,1]}}', None),
('{"version":1,"partitions":{"0":[3,1],"1":[1,2]}}', None)
]
cluster = Cluster.create_from_zookeeper('zkconnect')
assert len(cluster.brokers) == 3
b1 = cluster.brokers[1]
b3 = cluster.brokers[3]
assert cluster.brokers[3].hostname is None
assert cluster.topics['testTopic1'].partitions[1].replicas == [b3, b1]
assert cluster.topics['testTopic2'].partitions[0].replicas == [b3, b1]
def main():
# Start by loading all the modules
action_map = get_action_map()
sizer_map = get_sizer_map()
plugins_list = get_plugins_list()
# Instantiate all plugins
plugins = [plugin() for plugin in plugins_list]
# Set up and parse all CLI arguments
args = set_up_arguments(action_map, sizer_map, plugins)
for plugin in plugins:
plugin.set_arguments(args)
tools_path = get_tools_path(args.tools_path)
check_java_home()
cluster = Cluster.create_from_zookeeper(args.zookeeper)
for plugin in plugins:
plugin.set_cluster(cluster)
# If the module needs the partition sizes, call a size module to get the information
check_and_get_sizes(action_map[args.action], args, cluster, sizer_map)
for plugin in plugins:
plugin.after_sizes()
if args.leadership:
log.info("Cluster Leadership Balance (before):")
cluster.log_broker_summary()
# Clone the cluster, and run the action to generate a new cluster state
newcluster = cluster.clone()
action_to_run = action_map[args.action](args, newcluster)
action_to_run.process_cluster()
for plugin in plugins:
plugin.set_new_cluster(action_to_run.cluster)
if args.leadership:
log.info("Cluster Leadership Balance (after):")
newcluster.log_broker_summary()
move_partitions = cluster.changed_partitions(action_to_run.cluster)
batches = split_partitions_into_batches(move_partitions, batch_size=args.moves, use_class=Reassignment)
for plugin in plugins:
plugin.set_batches(batches)
log.info("Partition moves required: {0}".format(len(move_partitions)))
log.info("Number of batches: {0}".format(len(batches)))
dry_run = args.generate or not args.execute
if dry_run:
log.info("--execute flag NOT specified. DRY RUN ONLY")
for i, batch in enumerate(batches):
log.info("Executing partition reassignment {0}/{1}: {2}".format(i + 1, len(batches), repr(batch)))
batch.execute(i + 1, len(batches), args.zookeeper, tools_path, plugins, dry_run)
for plugin in plugins:
plugin.before_ple()
if not args.skip_ple:
batches = split_partitions_into_batches(move_partitions, batch_size=args.moves, use_class=ReplicaElection)
log.info("Number of replica elections: {0}".format(len(batches)))
run_preferred_replica_elections(batches, args, tools_path, plugins, dry_run)
for plugin in plugins:
plugin.finished()
return 0
