def connect(self): """ Connect to the all cluster brokers and populate topic and partition information. If the client was created with a zkconnect string, first we connect to Zookeeper to bootstrap the broker and topic information from there, and then the client connects to all brokers in the cluster. Otherwise, connect to the bootstrap broker specified and fetch the broker and topic metadata for the cluster. Todo: * Currently assumes everything works. Need to check for failure to connect to ZK or bootstrap. """ if self.configuration.zkconnect is not None: self.cluster = Cluster.create_from_zookeeper(zkconnect=self.configuration.zkconnect, fetch_topics=False) self._connected = True else: # Connect to bootstrap brokers until we succeed or exhaust the list try_brokers = list(self.configuration.broker_list) self._connected = False while (len(try_brokers) > 0) and not self._connected: self._connected = self._maybe_bootstrap_cluster(try_brokers.pop()) if not self._connected: raise ConnectionError("Unable to bootstrap cluster information") # Connect to all brokers self._connect_all_brokers()
def connect(self): """ Connect to the all cluster brokers and populate topic and partition information. If the client was created with a zkconnect string, first we connect to Zookeeper to bootstrap the broker and topic information from there, and then the client connects to all brokers in the cluster. Otherwise, connect to the bootstrap broker specified and fetch the broker and topic metadata for the cluster. Todo: * Currently assumes everything works. Need to check for failure to connect to ZK or bootstrap. """ if self.configuration.zkconnect is not None: self.cluster = Cluster.create_from_zookeeper( zkconnect=self.configuration.zkconnect, fetch_topics=False) self._connected = True else: # Connect to bootstrap brokers until we succeed or exhaust the list try_brokers = list(self.configuration.broker_list) self._connected = False while (len(try_brokers) > 0) and not self._connected: self._connected = self._maybe_bootstrap_cluster( try_brokers.pop()) if not self._connected: raise ConnectionError( "Unable to bootstrap cluster information") # Connect to all brokers self._connect_all_brokers()
def main(): # Start by loading all the modules action_map = get_module_map(kafka.tools.assigner.actions, kafka.tools.assigner.actions.ActionModule) sizer_map = get_module_map(kafka.tools.assigner.sizers, kafka.tools.assigner.sizers.SizerModule) plugins = get_all_plugins() # Set up and parse all CLI arguments args = set_up_arguments(action_map, sizer_map, plugins) run_plugins_at_step(plugins, 'set_arguments', args) tools_path = get_tools_path(args.tools_path) check_java_home() cluster = Cluster.create_from_zookeeper(args.zookeeper, getattr(args, 'default_retention', 1)) run_plugins_at_step(plugins, 'set_cluster', cluster) # If the module needs the partition sizes, call a size module to get the information check_and_get_sizes(action_map[args.action], args, cluster, sizer_map) run_plugins_at_step(plugins, 'after_sizes') print_leadership("before", cluster, args.leadership) # Clone the cluster, and run the action to generate a new cluster state newcluster = cluster.clone() action_to_run = action_map[args.action](args, newcluster) action_to_run.process_cluster() run_plugins_at_step(plugins, 'set_new_cluster', action_to_run.cluster) print_leadership("after", newcluster, args.leadership) move_partitions = cluster.changed_partitions(action_to_run.cluster) batches = split_partitions_into_batches(move_partitions, batch_size=args.moves, use_class=Reassignment) run_plugins_at_step(plugins, 'set_batches', batches) log.info("Partition moves required: {0}".format(len(move_partitions))) log.info("Number of batches: {0}".format(len(batches))) dry_run = is_dry_run(args) for i, batch in enumerate(batches): log.info("Executing partition reassignment {0}/{1}: {2}".format(i + 1, len(batches), repr(batch))) batch.execute(i + 1, len(batches), args.zookeeper, tools_path, plugins, dry_run) run_plugins_at_step(plugins, 'before_ple') if not args.skip_ple: all_cluster_partitions = [p for p in action_to_run.cluster.partitions(args.exclude_topics)] batches = split_partitions_into_batches(all_cluster_partitions, batch_size=args.ple_size, use_class=ReplicaElection) log.info("Number of replica elections: {0}".format(len(batches))) run_preferred_replica_elections(batches, args, tools_path, plugins, dry_run) run_plugins_at_step(plugins, 'finished') if args.output_json: data = { 'before': cluster.to_dict(), 'after': action_to_run.cluster.to_dict() } sys.stdout.write(json.dumps(data, indent=4, sort_keys=True)) return os.EX_OK
def test_cluster_create_with_retention(self): self.mock_children.side_effect = [['1', '2'], ['testTopic1', 'testTopic2']] self.mock_get.side_effect = [ (('{"jmx_port":7667,"timestamp":"1465289114807","endpoints":["PLAINTEXT://brokerhost1.example.com:9223",' '"SSL://brokerhost1.example.com:9224"],"host":"brokerhost1.example.com","version":1,"port":9223}' ), None), (('{"jmx_port":7667,"timestamp":"1465289114807","endpoints":["PLAINTEXT://brokerhost2.example.com:9223",' '"SSL://brokerhost2.example.com:9224"],"host":"brokerhost2.example.com","version":1,"port":9223}' ), None), ('{"version":1,"partitions":{"0":[1,2],"1":[2,1]}}', None), ('{"version":1,"config":{}}', None), ('{"version":1,"partitions":{"0":[2,1],"1":[1,2]}}', None), ('{"version":1,"config":{"retention.ms":"172800000"}}', None) ] cluster = Cluster.create_from_zookeeper('zkconnect') assert cluster.topics['testTopic1'].retention == 1 assert cluster.topics['testTopic2'].retention == 172800000
def test_cluster_create_missing_broker(self): self.mock_children.side_effect = [['1', '2'], ['testTopic1', 'testTopic2']] self.mock_get.side_effect = [ (('{"jmx_port":7667,"timestamp":"1465289114807","endpoints":["PLAINTEXT://brokerhost1.example.com:9223",' '"SSL://brokerhost1.example.com:9224"],"host":"brokerhost1.example.com","version":1,"port":9223}' ), None), (('{"jmx_port":7667,"timestamp":"1465289114807","endpoints":["PLAINTEXT://brokerhost2.example.com:9223",' '"SSL://brokerhost2.example.com:9224"],"host":"brokerhost2.example.com","version":1,"port":9223}' ), None), ('{"version":1,"partitions":{"0":[1,2],"1":[3,1]}}', None), ('{"version":1,"config":{}}', None), ('{"version":1,"partitions":{"0":[3,1],"1":[1,2]}}', None), ('{"version":1,"config":{}}', None) ] cluster = Cluster.create_from_zookeeper('zkconnect') assert len(cluster.brokers) == 3 b1 = cluster.brokers[1] b3 = cluster.brokers[3] assert cluster.brokers[3].hostname is None assert cluster.topics['testTopic1'].partitions[1].replicas == [b3, b1] assert cluster.topics['testTopic2'].partitions[0].replicas == [b3, b1]
def test_cluster_create_from_zookeeper(self): self.mock_children.side_effect = [['1', '2'], ['testTopic1', 'testTopic2']] self.mock_get.side_effect = [ (('{"jmx_port":7667,"timestamp":"1465289114807","endpoints":["PLAINTEXT://brokerhost1.example.com:9223",' '"SSL://brokerhost1.example.com:9224"],"host":"brokerhost1.example.com","version":1,"port":9223}' ), None), (('{"jmx_port":7667,"timestamp":"1465289114807","endpoints":["PLAINTEXT://brokerhost2.example.com:9223",' '"SSL://brokerhost2.example.com:9224"],"host":"brokerhost2.example.com","version":1,"port":9223}' ), None), ('{"version":1,"partitions":{"0":[1,2],"1":[2,1]}}', None), ('{"version":1,"config":{}}', None), ('{"version":1,"partitions":{"0":[2,1],"1":[1,2]}}', None), ('{"version":1,"config":{}}', None) ] cluster = Cluster.create_from_zookeeper('zkconnect') assert len(cluster.brokers) == 2 b1 = cluster.brokers[1] b2 = cluster.brokers[2] assert b1.hostname == 'brokerhost1.example.com' assert b2.hostname == 'brokerhost2.example.com' assert len(cluster.topics) == 2 assert 'testTopic1' in cluster.topics assert 'testTopic2' in cluster.topics t1 = cluster.topics['testTopic1'] assert len(t1.partitions) == 2 assert t1.partitions[0].replicas == [b1, b2] assert t1.partitions[1].replicas == [b2, b1] t2 = cluster.topics['testTopic2'] assert len(t2.partitions) == 2 assert t2.partitions[0].replicas == [b2, b1] assert t2.partitions[1].replicas == [b1, b2]
def main(): # Start by loading all the modules action_map = get_module_map(kafka.tools.assigner.actions, kafka.tools.assigner.actions.ActionModule) sizer_map = get_module_map(kafka.tools.assigner.sizers, kafka.tools.assigner.sizers.SizerModule) plugins = get_all_plugins() # Set up and parse all CLI arguments args = set_up_arguments(action_map, sizer_map, plugins) run_plugins_at_step(plugins, 'set_arguments', args) tools_path = get_tools_path(args.tools_path) check_java_home() cluster = Cluster.create_from_zookeeper( args.zookeeper, getattr(args, 'default_retention', 1)) run_plugins_at_step(plugins, 'set_cluster', cluster) # If the module needs the partition sizes, call a size module to get the information check_and_get_sizes(action_map[args.action], args, cluster, sizer_map) run_plugins_at_step(plugins, 'after_sizes') print_leadership("before", cluster, args.leadership) # Clone the cluster, and run the action to generate a new cluster state newcluster = cluster.clone() action_to_run = action_map[args.action](args, newcluster) action_to_run.process_cluster() run_plugins_at_step(plugins, 'set_new_cluster', action_to_run.cluster) print_leadership("after", newcluster, args.leadership) move_partitions = cluster.changed_partitions(action_to_run.cluster) batches = split_partitions_into_batches(move_partitions, batch_size=args.moves, use_class=Reassignment) run_plugins_at_step(plugins, 'set_batches', batches) log.info("Partition moves required: {0}".format(len(move_partitions))) log.info("Number of batches: {0}".format(len(batches))) dry_run = is_dry_run(args) for i, batch in enumerate(batches): log.info("Executing partition reassignment {0}/{1}: {2}".format( i + 1, len(batches), repr(batch))) batch.execute(i + 1, len(batches), args.zookeeper, tools_path, plugins, dry_run) run_plugins_at_step(plugins, 'before_ple') if not args.skip_ple: all_cluster_partitions = [ p for p in action_to_run.cluster.partitions(args.exclude_topics) ] batches = split_partitions_into_batches(all_cluster_partitions, batch_size=args.ple_size, use_class=ReplicaElection) log.info("Number of replica elections: {0}".format(len(batches))) run_preferred_replica_elections(batches, args, tools_path, plugins, dry_run) run_plugins_at_step(plugins, 'finished') return os.EX_OK