def __check_all_broker_ids_exist(broker_ids: list, zk: BukuExhibitor):
registered_brokers = zk.get_broker_ids()
unknown_brokers = [broker_id for broker_id in broker_ids if broker_id not in registered_brokers]
if len(unknown_brokers) == 1:
raise Exception('1 broker id is not valid: {}'.format(unknown_brokers[0]))
if len(unknown_brokers) > 1:
raise Exception('{} broker ids are not valid: {}'.format(len(unknown_brokers), ",".join(unknown_brokers)))
def __check_all_broker_ids_exist(broker_ids: list, zk: BukuExhibitor):
registered_brokers = zk.get_broker_ids()
unknown_brokers = [broker_id for broker_id in broker_ids if broker_id not in registered_brokers]
if len(unknown_brokers) == 1:
raise Exception('1 broker id is not valid: {}'.format(unknown_brokers[0]))
if len(unknown_brokers) > 1:
raise Exception('{} broker ids are not valid: {}'.format(len(unknown_brokers), ",".join(unknown_brokers)))
def register_migration(zk: BukuExhibitor, brokers_from: list, brokers_to: list, shrink: bool, broker_id: str,
parallelism: int):
if len(brokers_from) != len(brokers_to):
raise Exception('Brokers list {} and {} must have the same size'.format(brokers_from, brokers_to))
if any(b in brokers_from for b in brokers_to) or any(b in brokers_to for b in brokers_from):
raise Exception('Broker lists can not hold same broker ids')
if len(set(brokers_from)) != len(brokers_from):
raise Exception('Can not use same broker ids for source_list {}'.format(brokers_from))
if len(set(brokers_to)) != len(brokers_to):
raise Exception('Can not use same broker ids for source_list {}'.format(brokers_from))
active_ids = zk.get_broker_ids()
if any(b not in active_ids for b in brokers_from) or any(b not in active_ids for b in brokers_to):
raise Exception('Brokers dead from: {} to: {} alive:{}'.format(brokers_from, brokers_to, active_ids))
if broker_id and str(broker_id) not in active_ids:
raise Exception('Broker id to run change on ({}) is not in active list {}'.format(
broker_id, active_ids))
if parallelism <= 0:
raise Exception('Parallelism for migration should be greater than 0')
with zk.lock():
action = {'name': 'migrate', 'from': brokers_from, 'to': brokers_to, 'shrink': bool(shrink),
'parallelism': int(parallelism)}
if broker_id:
zk.register_action(action, str(broker_id))
else:
zk.register_action(action)
def register_rolling_restart(zk: BukuExhibitor, broker_id: str, image: str, instance_type: str, scalyr_key: str,
scalyr_region: str, kms_key_id: str, cool_down: int):
if zk.is_rolling_restart_in_progress():
_LOG.warning('Rolling restart in progress, skipping')
return
restart_assignment = {}
brokers = zk.get_broker_ids()
for idx in range(len(brokers)):
broker_to_make_restart = brokers[idx]
if idx == len(brokers) - 1:
broker_to_restart = brokers[0]
else:
broker_to_restart = brokers[idx + 1]
restart_assignment[broker_to_make_restart] = broker_to_restart
_LOG.info('Rolling restart assignment\n {}'.format(restart_assignment))
action = {'name': 'rolling_restart',
'restart_assignment': restart_assignment,
'image': image,
'instance_type': instance_type,
'scalyr_key': scalyr_key,
'scalyr_region': scalyr_region,
'kms_key_id': kms_key_id,
'cool_down': cool_down}
zk.register_action(action, broker_id=broker_id)
def register_migration(zk: BukuExhibitor, brokers_from: list, brokers_to: list, shrink: bool, broker_id: str,
throttle: int, parallelism: int):
if len(brokers_from) != len(brokers_to):
raise Exception('Brokers list {} and {} must have the same size'.format(brokers_from, brokers_to))
if any(b in brokers_from for b in brokers_to) or any(b in brokers_to for b in brokers_from):
raise Exception('Broker lists can not hold same broker ids')
if len(set(brokers_from)) != len(brokers_from):
raise Exception('Can not use same broker ids for source_list {}'.format(brokers_from))
if len(set(brokers_to)) != len(brokers_to):
raise Exception('Can not use same broker ids for source_list {}'.format(brokers_from))
active_ids = zk.get_broker_ids()
if any(b not in active_ids for b in brokers_from) or any(b not in active_ids for b in brokers_to):
raise Exception('Brokers dead from: {} to: {} alive:{}'.format(brokers_from, brokers_to, active_ids))
if broker_id and str(broker_id) not in active_ids:
raise Exception('Broker id to run change on ({}) is not in active list {}'.format(
broker_id, active_ids))
if parallelism <= 0:
raise Exception('Parallelism for migration should be greater than 0')
with zk.lock():
action = {'name': 'migrate', 'from': brokers_from, 'to': brokers_to, 'shrink': bool(shrink),
'parallelism': int(parallelism), 'throttle': int(throttle)}
if broker_id:
zk.register_action(action, str(broker_id))
else:
zk.register_action(action)
def get_opt_broker_id(broker_id: str, config: Config, zk: BukuExhibitor, env_provider: EnvProvider) -> str:
if not broker_id:
kafka_properties = KafkaProperties(config.kafka_settings_template, '/tmp/tmp.props'.format(config.kafka_dir))
broker_id_manager = env_provider.create_broker_id_manager(zk, kafka_properties)
broker_id = broker_id_manager.get_broker_id()
_LOG.info('Will use broker_id {}'.format(broker_id))
running_brokers = zk.get_broker_ids()
if broker_id not in running_brokers:
raise Exception('Broker id {} is not registered ({})'.format(broker_id, running_brokers))
return broker_id
def __get_opt_broker_id(broker_id: str, config: Config, zk: BukuExhibitor, env_provider: EnvProvider) -> str:
if not broker_id:
kafka_properties = KafkaProperties(config.kafka_settings_template, '/tmp/tmp.props'.format(config.kafka_dir))
broker_id_manager = env_provider.create_broker_id_manager(zk, kafka_properties)
broker_id = broker_id_manager.detect_broker_id()
_LOG.info('Will use broker_id {}'.format(broker_id))
running_brokers = zk.get_broker_ids()
if broker_id not in running_brokers:
raise Exception('Broker id {} is not registered ({}), can not restart'.format(broker_id, running_brokers))
return broker_id
def test_get_broker_ids():
exhibitor_mock = MagicMock()
def _get_children(path):
if path == '/brokers/ids':
return ['3', '1', '2']
else:
raise NotImplementedError()
exhibitor_mock.get_children = _get_children
buku = BukuExhibitor(exhibitor_mock)
assert ['1', '2', '3'] == buku.get_broker_ids() # ensure that return list is sorted