def connect(self):
"""
Connect to the all cluster brokers and populate topic and partition information. If the client was created with
a zkconnect string, first we connect to Zookeeper to bootstrap the broker and topic information from there,
and then the client connects to all brokers in the cluster. Otherwise, connect to the bootstrap broker
specified and fetch the broker and topic metadata for the cluster.
Todo:
* Currently assumes everything works. Need to check for failure to connect to
ZK or bootstrap.
"""
if self.configuration.zkconnect is not None:
self.cluster = Cluster.create_from_zookeeper(zkconnect=self.configuration.zkconnect, fetch_topics=False)
self._connected = True
else:
# Connect to bootstrap brokers until we succeed or exhaust the list
try_brokers = list(self.configuration.broker_list)
self._connected = False
while (len(try_brokers) > 0) and not self._connected:
self._connected = self._maybe_bootstrap_cluster(try_brokers.pop())
if not self._connected:
raise ConnectionError("Unable to bootstrap cluster information")
# Connect to all brokers
self._connect_all_brokers()
def connect(self):
"""
Connect to the all cluster brokers and populate topic and partition information. If the client was created with
a zkconnect string, first we connect to Zookeeper to bootstrap the broker and topic information from there,
and then the client connects to all brokers in the cluster. Otherwise, connect to the bootstrap broker
specified and fetch the broker and topic metadata for the cluster.
Todo:
* Currently assumes everything works. Need to check for failure to connect to
ZK or bootstrap.
"""
if self.configuration.zkconnect is not None:
self.cluster = Cluster.create_from_zookeeper(
zkconnect=self.configuration.zkconnect, fetch_topics=False)
self._connected = True
else:
# Connect to bootstrap brokers until we succeed or exhaust the list
try_brokers = list(self.configuration.broker_list)
self._connected = False
while (len(try_brokers) > 0) and not self._connected:
self._connected = self._maybe_bootstrap_cluster(
try_brokers.pop())
if not self._connected:
raise ConnectionError(
"Unable to bootstrap cluster information")
# Connect to all brokers
self._connect_all_brokers()
def main():
# Start by loading all the modules
action_map = get_module_map(kafka.tools.assigner.actions, kafka.tools.assigner.actions.ActionModule)
sizer_map = get_module_map(kafka.tools.assigner.sizers, kafka.tools.assigner.sizers.SizerModule)
plugins = get_all_plugins()
# Set up and parse all CLI arguments
args = set_up_arguments(action_map, sizer_map, plugins)
run_plugins_at_step(plugins, 'set_arguments', args)
tools_path = get_tools_path(args.tools_path)
check_java_home()
cluster = Cluster.create_from_zookeeper(args.zookeeper, getattr(args, 'default_retention', 1))
run_plugins_at_step(plugins, 'set_cluster', cluster)
# If the module needs the partition sizes, call a size module to get the information
check_and_get_sizes(action_map[args.action], args, cluster, sizer_map)
run_plugins_at_step(plugins, 'after_sizes')
print_leadership("before", cluster, args.leadership)
# Clone the cluster, and run the action to generate a new cluster state
newcluster = cluster.clone()
action_to_run = action_map[args.action](args, newcluster)
action_to_run.process_cluster()
run_plugins_at_step(plugins, 'set_new_cluster', action_to_run.cluster)
print_leadership("after", newcluster, args.leadership)
move_partitions = cluster.changed_partitions(action_to_run.cluster)
batches = split_partitions_into_batches(move_partitions, batch_size=args.moves, use_class=Reassignment)
run_plugins_at_step(plugins, 'set_batches', batches)
log.info("Partition moves required: {0}".format(len(move_partitions)))
log.info("Number of batches: {0}".format(len(batches)))
dry_run = is_dry_run(args)
for i, batch in enumerate(batches):
log.info("Executing partition reassignment {0}/{1}: {2}".format(i + 1, len(batches), repr(batch)))
batch.execute(i + 1, len(batches), args.zookeeper, tools_path, plugins, dry_run)
run_plugins_at_step(plugins, 'before_ple')
if not args.skip_ple:
all_cluster_partitions = [p for p in action_to_run.cluster.partitions(args.exclude_topics)]
batches = split_partitions_into_batches(all_cluster_partitions, batch_size=args.ple_size, use_class=ReplicaElection)
log.info("Number of replica elections: {0}".format(len(batches)))
run_preferred_replica_elections(batches, args, tools_path, plugins, dry_run)
run_plugins_at_step(plugins, 'finished')
if args.output_json:
data = {
'before': cluster.to_dict(),
'after': action_to_run.cluster.to_dict()
}
sys.stdout.write(json.dumps(data, indent=4, sort_keys=True))
return os.EX_OK
def test_cluster_create_with_retention(self):
self.mock_children.side_effect = [['1', '2'],
['testTopic1', 'testTopic2']]
self.mock_get.side_effect = [
(('{"jmx_port":7667,"timestamp":"1465289114807","endpoints":["PLAINTEXT://brokerhost1.example.com:9223",'
'"SSL://brokerhost1.example.com:9224"],"host":"brokerhost1.example.com","version":1,"port":9223}'
), None),
(('{"jmx_port":7667,"timestamp":"1465289114807","endpoints":["PLAINTEXT://brokerhost2.example.com:9223",'
'"SSL://brokerhost2.example.com:9224"],"host":"brokerhost2.example.com","version":1,"port":9223}'
), None),
('{"version":1,"partitions":{"0":[1,2],"1":[2,1]}}', None),
('{"version":1,"config":{}}', None),
('{"version":1,"partitions":{"0":[2,1],"1":[1,2]}}', None),
('{"version":1,"config":{"retention.ms":"172800000"}}', None)
]
cluster = Cluster.create_from_zookeeper('zkconnect')
assert cluster.topics['testTopic1'].retention == 1
assert cluster.topics['testTopic2'].retention == 172800000
def test_cluster_create_missing_broker(self):
self.mock_children.side_effect = [['1', '2'],
['testTopic1', 'testTopic2']]
self.mock_get.side_effect = [
(('{"jmx_port":7667,"timestamp":"1465289114807","endpoints":["PLAINTEXT://brokerhost1.example.com:9223",'
'"SSL://brokerhost1.example.com:9224"],"host":"brokerhost1.example.com","version":1,"port":9223}'
), None),
(('{"jmx_port":7667,"timestamp":"1465289114807","endpoints":["PLAINTEXT://brokerhost2.example.com:9223",'
'"SSL://brokerhost2.example.com:9224"],"host":"brokerhost2.example.com","version":1,"port":9223}'
), None),
('{"version":1,"partitions":{"0":[1,2],"1":[3,1]}}', None),
('{"version":1,"config":{}}', None),
('{"version":1,"partitions":{"0":[3,1],"1":[1,2]}}', None),
('{"version":1,"config":{}}', None)
]
cluster = Cluster.create_from_zookeeper('zkconnect')
assert len(cluster.brokers) == 3
b1 = cluster.brokers[1]
b3 = cluster.brokers[3]
assert cluster.brokers[3].hostname is None
assert cluster.topics['testTopic1'].partitions[1].replicas == [b3, b1]
assert cluster.topics['testTopic2'].partitions[0].replicas == [b3, b1]
def test_cluster_create_from_zookeeper(self):
self.mock_children.side_effect = [['1', '2'],
['testTopic1', 'testTopic2']]
self.mock_get.side_effect = [
(('{"jmx_port":7667,"timestamp":"1465289114807","endpoints":["PLAINTEXT://brokerhost1.example.com:9223",'
'"SSL://brokerhost1.example.com:9224"],"host":"brokerhost1.example.com","version":1,"port":9223}'
), None),
(('{"jmx_port":7667,"timestamp":"1465289114807","endpoints":["PLAINTEXT://brokerhost2.example.com:9223",'
'"SSL://brokerhost2.example.com:9224"],"host":"brokerhost2.example.com","version":1,"port":9223}'
), None),
('{"version":1,"partitions":{"0":[1,2],"1":[2,1]}}', None),
('{"version":1,"config":{}}', None),
('{"version":1,"partitions":{"0":[2,1],"1":[1,2]}}', None),
('{"version":1,"config":{}}', None)
]
cluster = Cluster.create_from_zookeeper('zkconnect')
assert len(cluster.brokers) == 2
b1 = cluster.brokers[1]
b2 = cluster.brokers[2]
assert b1.hostname == 'brokerhost1.example.com'
assert b2.hostname == 'brokerhost2.example.com'
assert len(cluster.topics) == 2
assert 'testTopic1' in cluster.topics
assert 'testTopic2' in cluster.topics
t1 = cluster.topics['testTopic1']
assert len(t1.partitions) == 2
assert t1.partitions[0].replicas == [b1, b2]
assert t1.partitions[1].replicas == [b2, b1]
t2 = cluster.topics['testTopic2']
assert len(t2.partitions) == 2
assert t2.partitions[0].replicas == [b2, b1]
assert t2.partitions[1].replicas == [b1, b2]
def main():
# Start by loading all the modules
action_map = get_module_map(kafka.tools.assigner.actions,
kafka.tools.assigner.actions.ActionModule)
sizer_map = get_module_map(kafka.tools.assigner.sizers,
kafka.tools.assigner.sizers.SizerModule)
plugins = get_all_plugins()
# Set up and parse all CLI arguments
args = set_up_arguments(action_map, sizer_map, plugins)
run_plugins_at_step(plugins, 'set_arguments', args)
tools_path = get_tools_path(args.tools_path)
check_java_home()
cluster = Cluster.create_from_zookeeper(
args.zookeeper, getattr(args, 'default_retention', 1))
run_plugins_at_step(plugins, 'set_cluster', cluster)
# If the module needs the partition sizes, call a size module to get the information
check_and_get_sizes(action_map[args.action], args, cluster, sizer_map)
run_plugins_at_step(plugins, 'after_sizes')
print_leadership("before", cluster, args.leadership)
# Clone the cluster, and run the action to generate a new cluster state
newcluster = cluster.clone()
action_to_run = action_map[args.action](args, newcluster)
action_to_run.process_cluster()
run_plugins_at_step(plugins, 'set_new_cluster', action_to_run.cluster)
print_leadership("after", newcluster, args.leadership)
move_partitions = cluster.changed_partitions(action_to_run.cluster)
batches = split_partitions_into_batches(move_partitions,
batch_size=args.moves,
use_class=Reassignment)
run_plugins_at_step(plugins, 'set_batches', batches)
log.info("Partition moves required: {0}".format(len(move_partitions)))
log.info("Number of batches: {0}".format(len(batches)))
dry_run = is_dry_run(args)
for i, batch in enumerate(batches):
log.info("Executing partition reassignment {0}/{1}: {2}".format(
i + 1, len(batches), repr(batch)))
batch.execute(i + 1, len(batches), args.zookeeper, tools_path, plugins,
dry_run)
run_plugins_at_step(plugins, 'before_ple')
if not args.skip_ple:
all_cluster_partitions = [
p for p in action_to_run.cluster.partitions(args.exclude_topics)
]
batches = split_partitions_into_batches(all_cluster_partitions,
batch_size=args.ple_size,
use_class=ReplicaElection)
log.info("Number of replica elections: {0}".format(len(batches)))
run_preferred_replica_elections(batches, args, tools_path, plugins,
dry_run)
run_plugins_at_step(plugins, 'finished')
return os.EX_OK