def run_daemon_loop(config: Config, process_holder: KafkaProcess, cmd_helper: CmdHelper, restart_on_init: bool):
_LOG.info("Using configuration: {}".format(config))
kafka_props = KafkaProperties(config.kafka_settings_template,
'{}/config/server.properties'.format(config.kafka_dir))
env_provider = EnvProvider.create_env_provider(config)
address_provider = env_provider.get_address_provider()
rack = env_provider.get_rack()
if rack:
kafka_props.set_property('broker.rack', rack)
startup_timeout = StartupTimeout.build(config.timeout)
_LOG.info("Loading exhibitor configuration")
with load_exhibitor_proxy(address_provider, config.zk_prefix) as zookeeper:
_LOG.info("Loading broker_id policy")
broker_id_manager = env_provider.create_broker_id_manager(zookeeper, kafka_props)
_LOG.info("Building broker manager")
broker = BrokerManager(process_holder, zookeeper, broker_id_manager, kafka_props,
startup_timeout)
_LOG.info("Creating controller")
controller = Controller(broker, zookeeper, env_provider)
controller.add_check(CheckBrokerStopped(broker, zookeeper))
controller.add_check(RemoteCommandExecutorCheck(zookeeper, broker, config.health_port))
controller.add_check(GenerateDataSizeStatistics(zookeeper, broker, cmd_helper,
kafka_props.get_property("log.dirs").split(",")))
apply_features(config.health_port, config.features, controller, zookeeper, broker, kafka_props, env_provider)
_LOG.info('Starting main controller loop')
controller.loop(RestartBrokerChange(zookeeper, broker, lambda: False) if restart_on_init else None)
def rebalance_partitions(broker: str):
config, env_provider = __prepare_configs()
with load_exhibitor_proxy(env_provider.get_address_provider(),
config.zk_prefix) as zookeeper:
broker_id = __get_opt_broker_id(broker, config, zookeeper,
env_provider) if broker else None
RemoteCommandExecutorCheck.register_rebalance(zookeeper, broker_id)
def _list_broker_addresses(config, env_provider, broker):
with load_exhibitor_proxy(env_provider.get_address_provider(),
config.zk_prefix) as zookeeper:
for broker_id in zookeeper.get_broker_ids():
if broker and broker != broker_id:
continue
yield broker_id, zookeeper.get_broker_address(broker_id)
def migrate_broker(from_: str, to: str, shrink: bool, broker: str, throttle: int, parallelism: int,
remove_throttle: bool):
config, env_provider = prepare_configs()
with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper:
if remove_throttle:
return RebalanceThrottleManager.remove_all_throttle_configurations(zookeeper)
broker_id = get_opt_broker_id(broker, config, zookeeper, env_provider) if broker else None
RemoteCommandExecutorCheck.register_migration(zookeeper, from_.split(','), to.split(','), shrink, broker_id,
throttle, parallelism)
def rebalance_partitions(broker: str, empty_brokers: str, exclude_topics: str):
config, env_provider = __prepare_configs()
with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper:
empty_brokers_list = [] if empty_brokers is None else empty_brokers.split(',')
exclude_topics_list = [] if exclude_topics is None else exclude_topics.split(',')
__check_all_broker_ids_exist(empty_brokers_list, zookeeper)
broker_id = __get_opt_broker_id(broker, config, zookeeper, env_provider) if broker else None
RemoteCommandExecutorCheck.register_rebalance(zookeeper, broker_id, empty_brokers_list,
exclude_topics_list)
def migrate_broker(from_: str, to: str, shrink: bool, broker: str):
config, env_provider = __prepare_configs()
with load_exhibitor_proxy(env_provider.get_address_provider(),
config.zk_prefix) as zookeeper:
broker_id = __get_opt_broker_id(broker, config, zookeeper,
env_provider) if broker else None
RemoteCommandExecutorCheck.register_migration(zookeeper,
from_.split(','),
to.split(','), shrink,
broker_id)
def rolling_restart_broker(image_tag: str, instance_type: str, scalyr_key: str, scalyr_region: str, kms_key_id: str,
cool_down: int):
if not is_cluster_healthy():
print('Cluster is not healthy, try again later :)')
return
config, env_provider = prepare_configs()
with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper:
broker_id = get_opt_broker_id(None, config, zookeeper, env_provider)
RemoteCommandExecutorCheck.register_rolling_restart(zookeeper, broker_id, image_tag, instance_type, scalyr_key,
scalyr_region, kms_key_id, cool_down)
def show_stats():
config, env_provider = __prepare_configs()
with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper:
disk_stats = zookeeper.get_disk_stats()
table = []
for broker_id in zookeeper.get_broker_ids():
disk = disk_stats.get(broker_id, {}).get('disk') if disk_stats else {}
table.append({
'Broker Id': broker_id,
'Address': zookeeper.get_broker_address(broker_id),
'Free kb': disk.get('free_kb'),
'Used kb': disk.get('used_kb')
})
_print_table(table)
def do_GET(self):
if self.path in ('/api/disk_stats', '/api/disk_stats/'):
used_kb, free_kb = self.cmd_helper.get_disk_stats()
self._send_response({'free_kb': free_kb, 'used_kb': used_kb})
elif self.path.startswith(_API_CONTROLLER):
self.wrap_controller_execution(lambda: self._run_controller_action(self.path[len(_API_CONTROLLER):]))
elif self.path in ('/api/metrics', '/api/metrics/'):
config = load_config()
env_provider = EnvProvider.create_env_provider(config)
with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper:
metrics = MetricCollector(zookeeper).get_metrics_from_brokers()
self._send_response({'metrics': metrics})
else:
self._send_response({'status': 'OK'})
def show_stats():
config, env_provider = prepare_configs()
with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper:
disk_stats = zookeeper.get_disk_stats()
table = []
for broker_id in zookeeper.get_broker_ids():
disk = disk_stats.get(broker_id, {}).get('disk') if disk_stats else {}
table.append({
'Broker Id': broker_id,
'Address': zookeeper.get_broker_address(broker_id),
'Free kb': disk.get('free_kb'),
'Used kb': disk.get('used_kb')
})
_print_table(table)
def rebalance_partitions(broker: str, empty_brokers: str, exclude_topics: str, parallelism: int, bin_packing: bool,
throttle: int, remove_throttle: bool):
if throttle and throttle < get_max_bytes_in():
print('Throttle value must be set above the max BytesIn for the replication to progress. '
'The current max BytesIn is {}'.format(get_max_bytes_in()))
exit(1)
config, env_provider = prepare_configs()
with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper:
if remove_throttle:
return RebalanceThrottleManager.remove_all_throttle_configurations(zookeeper)
empty_brokers_list = [] if empty_brokers is None else empty_brokers.split(',')
exclude_topics_list = [] if exclude_topics is None else exclude_topics.split(',')
__check_all_broker_ids_exist(empty_brokers_list, zookeeper)
broker_id = get_opt_broker_id(broker, config, zookeeper, env_provider) if broker else None
RemoteCommandExecutorCheck.register_rebalance(zookeeper, broker_id, empty_brokers_list,
exclude_topics_list, parallelism, bin_packing, throttle)
def validate_replication(factor: int):
config, env_provider = prepare_configs()
with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper:
brokers = {int(x) for x in zookeeper.get_broker_ids()}
table = []
for topic_name, partition, state in zookeeper.load_partition_states():
if len(state['isr']) != factor or not set(state['isr']).issubset(brokers):
table.append({
'Partition': partition,
'Topic': topic_name,
'State': state
})
if table:
_LOG.info('Invalid topics:')
_print_table(table)
else:
print('All replica lists look valid')
def validate_replication(factor: int):
config, env_provider = __prepare_configs()
with load_exhibitor_proxy(env_provider.get_address_provider(),
config.zk_prefix) as zookeeper:
brokers = {int(x) for x in zookeeper.get_broker_ids()}
table = []
for topic_name, partition, state in zookeeper.load_partition_states():
if len(state['isr']) != factor or not set(
state['isr']).issubset(brokers):
table.append({
'Partition': partition,
'Topic': topic_name,
'State': state
})
if table:
_LOG.info('Invalid topics:')
_print_table(table)
else:
print('All replica lists look valid')
def run_daemon_loop(config: Config, process_holder: KafkaProcess,
cmd_helper: CmdHelper, restart_on_init: bool):
_LOG.info("Using configuration: {}".format(config))
kafka_props = KafkaProperties(
config.kafka_settings_template,
'{}/config/server.properties'.format(config.kafka_dir))
env_provider = EnvProvider.create_env_provider(config)
address_provider = env_provider.get_address_provider()
rack = env_provider.get_rack()
if rack:
kafka_props.set_property('broker.rack', rack)
startup_timeout = StartupTimeout.build(config.timeout)
_LOG.info("Loading exhibitor configuration")
with load_exhibitor_proxy(address_provider, config.zk_prefix) as zookeeper:
_LOG.info("Loading broker_id policy")
broker_id_manager = env_provider.create_broker_id_manager(
zookeeper, kafka_props)
_LOG.info("Building broker manager")
broker = BrokerManager(process_holder, zookeeper, broker_id_manager,
kafka_props, startup_timeout)
_LOG.info("Creating controller")
controller = Controller(broker, zookeeper, env_provider)
controller.add_check(CheckBrokerStopped(broker, zookeeper))
controller.add_check(
RemoteCommandExecutorCheck(zookeeper, broker, config.health_port))
controller.add_check(
GenerateDataSizeStatistics(
zookeeper, broker, cmd_helper,
kafka_props.get_property("log.dirs").split(",")))
apply_features(config.health_port, config.features, controller,
zookeeper, broker, kafka_props, env_provider)
_LOG.info('Starting main controller loop')
controller.loop(
RestartBrokerChange(zookeeper, broker, lambda: False
) if restart_on_init else None)
def restart_broker(broker: str):
config, env_provider = prepare_configs()
with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper:
broker_id = get_opt_broker_id(broker, config, zookeeper, env_provider)
RemoteCommandExecutorCheck.register_restart(zookeeper, broker_id)
def swap_partitions(threshold: int):
config, env_provider = prepare_configs()
with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper:
RemoteCommandExecutorCheck.register_fatboy_slim(zookeeper, threshold_kb=threshold)
def swap_partitions(threshold: int):
config, env_provider = __prepare_configs()
with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper:
RemoteCommandExecutorCheck.register_fatboy_slim(zookeeper, threshold_kb=threshold)
def _list_broker_addresses(config, env_provider, broker):
with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper:
for broker_id in zookeeper.get_broker_ids():
if broker and broker != broker_id:
continue
yield broker_id, zookeeper.get_broker_address(broker_id)
