def run_daemon_loop(config: Config, process_holder: KafkaProcess, cmd_helper: CmdHelper, restart_on_init: bool): _LOG.info("Using configuration: {}".format(config)) kafka_props = KafkaProperties(config.kafka_settings_template, '{}/config/server.properties'.format(config.kafka_dir)) env_provider = EnvProvider.create_env_provider(config) address_provider = env_provider.get_address_provider() rack = env_provider.get_rack() if rack: kafka_props.set_property('broker.rack', rack) startup_timeout = StartupTimeout.build(config.timeout) _LOG.info("Loading exhibitor configuration") with load_exhibitor_proxy(address_provider, config.zk_prefix) as zookeeper: _LOG.info("Loading broker_id policy") broker_id_manager = env_provider.create_broker_id_manager(zookeeper, kafka_props) _LOG.info("Building broker manager") broker = BrokerManager(process_holder, zookeeper, broker_id_manager, kafka_props, startup_timeout) _LOG.info("Creating controller") controller = Controller(broker, zookeeper, env_provider) controller.add_check(CheckBrokerStopped(broker, zookeeper)) controller.add_check(RemoteCommandExecutorCheck(zookeeper, broker, config.health_port)) controller.add_check(GenerateDataSizeStatistics(zookeeper, broker, cmd_helper, kafka_props.get_property("log.dirs").split(","))) apply_features(config.health_port, config.features, controller, zookeeper, broker, kafka_props, env_provider) _LOG.info('Starting main controller loop') controller.loop(RestartBrokerChange(zookeeper, broker, lambda: False) if restart_on_init else None)
def rebalance_partitions(broker: str): config, env_provider = __prepare_configs() with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper: broker_id = __get_opt_broker_id(broker, config, zookeeper, env_provider) if broker else None RemoteCommandExecutorCheck.register_rebalance(zookeeper, broker_id)
def _list_broker_addresses(config, env_provider, broker): with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper: for broker_id in zookeeper.get_broker_ids(): if broker and broker != broker_id: continue yield broker_id, zookeeper.get_broker_address(broker_id)
def migrate_broker(from_: str, to: str, shrink: bool, broker: str, throttle: int, parallelism: int, remove_throttle: bool): config, env_provider = prepare_configs() with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper: if remove_throttle: return RebalanceThrottleManager.remove_all_throttle_configurations(zookeeper) broker_id = get_opt_broker_id(broker, config, zookeeper, env_provider) if broker else None RemoteCommandExecutorCheck.register_migration(zookeeper, from_.split(','), to.split(','), shrink, broker_id, throttle, parallelism)
def rebalance_partitions(broker: str, empty_brokers: str, exclude_topics: str): config, env_provider = __prepare_configs() with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper: empty_brokers_list = [] if empty_brokers is None else empty_brokers.split(',') exclude_topics_list = [] if exclude_topics is None else exclude_topics.split(',') __check_all_broker_ids_exist(empty_brokers_list, zookeeper) broker_id = __get_opt_broker_id(broker, config, zookeeper, env_provider) if broker else None RemoteCommandExecutorCheck.register_rebalance(zookeeper, broker_id, empty_brokers_list, exclude_topics_list)
def migrate_broker(from_: str, to: str, shrink: bool, broker: str): config, env_provider = __prepare_configs() with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper: broker_id = __get_opt_broker_id(broker, config, zookeeper, env_provider) if broker else None RemoteCommandExecutorCheck.register_migration(zookeeper, from_.split(','), to.split(','), shrink, broker_id)
def rolling_restart_broker(image_tag: str, instance_type: str, scalyr_key: str, scalyr_region: str, kms_key_id: str, cool_down: int): if not is_cluster_healthy(): print('Cluster is not healthy, try again later :)') return config, env_provider = prepare_configs() with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper: broker_id = get_opt_broker_id(None, config, zookeeper, env_provider) RemoteCommandExecutorCheck.register_rolling_restart(zookeeper, broker_id, image_tag, instance_type, scalyr_key, scalyr_region, kms_key_id, cool_down)
def show_stats(): config, env_provider = __prepare_configs() with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper: disk_stats = zookeeper.get_disk_stats() table = [] for broker_id in zookeeper.get_broker_ids(): disk = disk_stats.get(broker_id, {}).get('disk') if disk_stats else {} table.append({ 'Broker Id': broker_id, 'Address': zookeeper.get_broker_address(broker_id), 'Free kb': disk.get('free_kb'), 'Used kb': disk.get('used_kb') }) _print_table(table)
def do_GET(self): if self.path in ('/api/disk_stats', '/api/disk_stats/'): used_kb, free_kb = self.cmd_helper.get_disk_stats() self._send_response({'free_kb': free_kb, 'used_kb': used_kb}) elif self.path.startswith(_API_CONTROLLER): self.wrap_controller_execution(lambda: self._run_controller_action(self.path[len(_API_CONTROLLER):])) elif self.path in ('/api/metrics', '/api/metrics/'): config = load_config() env_provider = EnvProvider.create_env_provider(config) with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper: metrics = MetricCollector(zookeeper).get_metrics_from_brokers() self._send_response({'metrics': metrics}) else: self._send_response({'status': 'OK'})
def show_stats(): config, env_provider = prepare_configs() with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper: disk_stats = zookeeper.get_disk_stats() table = [] for broker_id in zookeeper.get_broker_ids(): disk = disk_stats.get(broker_id, {}).get('disk') if disk_stats else {} table.append({ 'Broker Id': broker_id, 'Address': zookeeper.get_broker_address(broker_id), 'Free kb': disk.get('free_kb'), 'Used kb': disk.get('used_kb') }) _print_table(table)
def rebalance_partitions(broker: str, empty_brokers: str, exclude_topics: str, parallelism: int, bin_packing: bool, throttle: int, remove_throttle: bool): if throttle and throttle < get_max_bytes_in(): print('Throttle value must be set above the max BytesIn for the replication to progress. ' 'The current max BytesIn is {}'.format(get_max_bytes_in())) exit(1) config, env_provider = prepare_configs() with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper: if remove_throttle: return RebalanceThrottleManager.remove_all_throttle_configurations(zookeeper) empty_brokers_list = [] if empty_brokers is None else empty_brokers.split(',') exclude_topics_list = [] if exclude_topics is None else exclude_topics.split(',') __check_all_broker_ids_exist(empty_brokers_list, zookeeper) broker_id = get_opt_broker_id(broker, config, zookeeper, env_provider) if broker else None RemoteCommandExecutorCheck.register_rebalance(zookeeper, broker_id, empty_brokers_list, exclude_topics_list, parallelism, bin_packing, throttle)
def validate_replication(factor: int): config, env_provider = prepare_configs() with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper: brokers = {int(x) for x in zookeeper.get_broker_ids()} table = [] for topic_name, partition, state in zookeeper.load_partition_states(): if len(state['isr']) != factor or not set(state['isr']).issubset(brokers): table.append({ 'Partition': partition, 'Topic': topic_name, 'State': state }) if table: _LOG.info('Invalid topics:') _print_table(table) else: print('All replica lists look valid')
def validate_replication(factor: int): config, env_provider = __prepare_configs() with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper: brokers = {int(x) for x in zookeeper.get_broker_ids()} table = [] for topic_name, partition, state in zookeeper.load_partition_states(): if len(state['isr']) != factor or not set( state['isr']).issubset(brokers): table.append({ 'Partition': partition, 'Topic': topic_name, 'State': state }) if table: _LOG.info('Invalid topics:') _print_table(table) else: print('All replica lists look valid')
def run_daemon_loop(config: Config, process_holder: KafkaProcess, cmd_helper: CmdHelper, restart_on_init: bool): _LOG.info("Using configuration: {}".format(config)) kafka_props = KafkaProperties( config.kafka_settings_template, '{}/config/server.properties'.format(config.kafka_dir)) env_provider = EnvProvider.create_env_provider(config) address_provider = env_provider.get_address_provider() rack = env_provider.get_rack() if rack: kafka_props.set_property('broker.rack', rack) startup_timeout = StartupTimeout.build(config.timeout) _LOG.info("Loading exhibitor configuration") with load_exhibitor_proxy(address_provider, config.zk_prefix) as zookeeper: _LOG.info("Loading broker_id policy") broker_id_manager = env_provider.create_broker_id_manager( zookeeper, kafka_props) _LOG.info("Building broker manager") broker = BrokerManager(process_holder, zookeeper, broker_id_manager, kafka_props, startup_timeout) _LOG.info("Creating controller") controller = Controller(broker, zookeeper, env_provider) controller.add_check(CheckBrokerStopped(broker, zookeeper)) controller.add_check( RemoteCommandExecutorCheck(zookeeper, broker, config.health_port)) controller.add_check( GenerateDataSizeStatistics( zookeeper, broker, cmd_helper, kafka_props.get_property("log.dirs").split(","))) apply_features(config.health_port, config.features, controller, zookeeper, broker, kafka_props, env_provider) _LOG.info('Starting main controller loop') controller.loop( RestartBrokerChange(zookeeper, broker, lambda: False ) if restart_on_init else None)
def restart_broker(broker: str): config, env_provider = prepare_configs() with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper: broker_id = get_opt_broker_id(broker, config, zookeeper, env_provider) RemoteCommandExecutorCheck.register_restart(zookeeper, broker_id)
def swap_partitions(threshold: int): config, env_provider = prepare_configs() with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper: RemoteCommandExecutorCheck.register_fatboy_slim(zookeeper, threshold_kb=threshold)
def swap_partitions(threshold: int): config, env_provider = __prepare_configs() with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper: RemoteCommandExecutorCheck.register_fatboy_slim(zookeeper, threshold_kb=threshold)
def _list_broker_addresses(config, env_provider, broker): with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper: for broker_id in zookeeper.get_broker_ids(): if broker and broker != broker_id: continue yield broker_id, zookeeper.get_broker_address(broker_id)