def rebalance_partitions(broker: str):
config, env_provider = __prepare_configs()
with load_exhibitor_proxy(env_provider.get_address_provider(),
config.zk_prefix) as zookeeper:
broker_id = __get_opt_broker_id(broker, config, zookeeper,
env_provider) if broker else None
RemoteCommandExecutorCheck.register_rebalance(zookeeper, broker_id)
def run(self):
if utils.is_cluster_healthy():
from bubuku.features.remote_exec import RemoteCommandExecutorCheck
RemoteCommandExecutorCheck.register_stop(self.state_context.zk, self.state_context.broker_id_to_restart)
return True
_LOG.warning('Cluster is not healthy, waiting for it to recover')
return False
def migrate_broker(from_: str, to: str, shrink: bool, broker: str, throttle: int, parallelism: int,
remove_throttle: bool):
config, env_provider = prepare_configs()
with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper:
if remove_throttle:
return RebalanceThrottleManager.remove_all_throttle_configurations(zookeeper)
broker_id = get_opt_broker_id(broker, config, zookeeper, env_provider) if broker else None
RemoteCommandExecutorCheck.register_migration(zookeeper, from_.split(','), to.split(','), shrink, broker_id,
throttle, parallelism)
def rebalance_partitions(broker: str, empty_brokers: str, exclude_topics: str):
config, env_provider = __prepare_configs()
with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper:
empty_brokers_list = [] if empty_brokers is None else empty_brokers.split(',')
exclude_topics_list = [] if exclude_topics is None else exclude_topics.split(',')
__check_all_broker_ids_exist(empty_brokers_list, zookeeper)
broker_id = __get_opt_broker_id(broker, config, zookeeper, env_provider) if broker else None
RemoteCommandExecutorCheck.register_rebalance(zookeeper, broker_id, empty_brokers_list,
exclude_topics_list)
def migrate_broker(from_: str, to: str, shrink: bool, broker: str):
config, env_provider = __prepare_configs()
with load_exhibitor_proxy(env_provider.get_address_provider(),
config.zk_prefix) as zookeeper:
broker_id = __get_opt_broker_id(broker, config, zookeeper,
env_provider) if broker else None
RemoteCommandExecutorCheck.register_migration(zookeeper,
from_.split(','),
to.split(','), shrink,
broker_id)
def rolling_restart_broker(image_tag: str, instance_type: str, scalyr_key: str, scalyr_region: str, kms_key_id: str,
cool_down: int):
if not is_cluster_healthy():
print('Cluster is not healthy, try again later :)')
return
config, env_provider = prepare_configs()
with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper:
broker_id = get_opt_broker_id(None, config, zookeeper, env_provider)
RemoteCommandExecutorCheck.register_rolling_restart(zookeeper, broker_id, image_tag, instance_type, scalyr_key,
scalyr_region, kms_key_id, cool_down)
def rebalance_partitions(broker: str, empty_brokers: str, exclude_topics: str, parallelism: int, bin_packing: bool,
throttle: int, remove_throttle: bool):
if throttle and throttle < get_max_bytes_in():
print('Throttle value must be set above the max BytesIn for the replication to progress. '
'The current max BytesIn is {}'.format(get_max_bytes_in()))
exit(1)
config, env_provider = prepare_configs()
with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper:
if remove_throttle:
return RebalanceThrottleManager.remove_all_throttle_configurations(zookeeper)
empty_brokers_list = [] if empty_brokers is None else empty_brokers.split(',')
exclude_topics_list = [] if exclude_topics is None else exclude_topics.split(',')
__check_all_broker_ids_exist(empty_brokers_list, zookeeper)
broker_id = get_opt_broker_id(broker, config, zookeeper, env_provider) if broker else None
RemoteCommandExecutorCheck.register_rebalance(zookeeper, broker_id, empty_brokers_list,
exclude_topics_list, parallelism, bin_packing, throttle)
def run_daemon_loop(config: Config, process_holder: KafkaProcess,
cmd_helper: CmdHelper, restart_on_init: bool):
_LOG.info("Using configuration: {}".format(config))
kafka_props = KafkaProperties(
config.kafka_settings_template,
'{}/config/server.properties'.format(config.kafka_dir))
env_provider = EnvProvider.create_env_provider(config)
address_provider = env_provider.get_address_provider()
rack = env_provider.get_rack()
if rack:
kafka_props.set_property('broker.rack', rack)
startup_timeout = StartupTimeout.build(config.timeout)
_LOG.info("Loading exhibitor configuration")
with load_exhibitor_proxy(address_provider, config.zk_prefix) as zookeeper:
_LOG.info("Loading broker_id policy")
broker_id_manager = env_provider.create_broker_id_manager(
zookeeper, kafka_props)
_LOG.info("Building broker manager")
broker = BrokerManager(process_holder, zookeeper, broker_id_manager,
kafka_props, startup_timeout)
_LOG.info("Creating controller")
controller = Controller(broker, zookeeper, env_provider)
controller.add_check(CheckBrokerStopped(broker, zookeeper))
controller.add_check(
RemoteCommandExecutorCheck(zookeeper, broker, config.health_port))
controller.add_check(
GenerateDataSizeStatistics(
zookeeper, broker, cmd_helper,
kafka_props.get_property("log.dirs").split(",")))
apply_features(config.health_port, config.features, controller,
zookeeper, broker, kafka_props, env_provider)
_LOG.info('Starting main controller loop')
controller.loop(
RestartBrokerChange(zookeeper, broker, lambda: False
) if restart_on_init else None)
def restart_broker(broker: str):
config, env_provider = prepare_configs()
with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper:
broker_id = get_opt_broker_id(broker, config, zookeeper, env_provider)
RemoteCommandExecutorCheck.register_restart(zookeeper, broker_id)
def swap_partitions(threshold: int):
config, env_provider = prepare_configs()
with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper:
RemoteCommandExecutorCheck.register_fatboy_slim(zookeeper, threshold_kb=threshold)
def swap_partitions(threshold: int):
config, env_provider = __prepare_configs()
with load_exhibitor_proxy(env_provider.get_address_provider(), config.zk_prefix) as zookeeper:
RemoteCommandExecutorCheck.register_fatboy_slim(zookeeper, threshold_kb=threshold)
def run(self):
from bubuku.features.remote_exec import RemoteCommandExecutorCheck
RemoteCommandExecutorCheck.register_start(self.state_context.zk, self.state_context.broker_id_to_restart)
return True