def get_subcloud_status(subcloud,
field='availability',
auth_info=Tenant.get('admin_platform', 'RegionOne'),
con_ssh=None,
source_openrc=None):
"""
Args:
subcloud:
field:
auth_info:
con_ssh:
source_openrc:
Returns:
"""
LOG.info("Auth_info: {}".format(auth_info))
table_ = table_parser.table(
cli.dcmanager('subcloud list',
ssh_client=con_ssh,
auth_info=auth_info,
source_openrc=source_openrc)[1])
arg_dict = {'name': subcloud}
kwargs = {key: val for key, val in arg_dict.items() if val is not None}
status = table_parser.get_values(table_, target_header=field, **kwargs)
return status[0]
def get_subclouds(field='name',
name=None,
avail=None,
sync=None,
mgmt=None,
deploy=None,
auth_info=Tenant.get('admin_platform', 'RegionOne'),
con_ssh=None,
source_openrc=None):
"""
Getting subclouds info
Args:
field:
name:
avail:
sync:
mgmt:
auth_info:
con_ssh:
source_openrc:
Returns:
"""
# auth_info = Tenant.get('admin', 'SystemController')
LOG.info("Auth_info: {}".format(auth_info))
table_ = table_parser.table(
cli.dcmanager('subcloud list',
ssh_client=con_ssh,
auth_info=auth_info,
source_openrc=source_openrc)[1])
arg_dict = {
'name': name,
'availability': avail,
'sync': sync,
'management': mgmt,
'deploy status': deploy
}
kwargs = {key: val for key, val in arg_dict.items() if val is not None}
subclouds = table_parser.get_values(table_, target_header=field, **kwargs)
return subclouds
def check_alarm_summary_match_subcloud(subcloud, timeout=400):
LOG.info(
"Ensure alarm summary on SystemController with subcloud {}".format(
subcloud))
subcloud_auth = Tenant.get('admin_platform', dc_region=subcloud)
central_auth = Tenant.get('admin_platform', dc_region='RegionOne')
severities = [
"critical_alarms", "major_alarms", "minor_alarms", "warnings"
]
central_alarms = subcloud_alarms = None
end_time = time.time() + timeout
while time.time() < end_time:
output_central = cli.dcmanager('alarm summary',
auth_info=central_auth,
fail_ok=False)[1]
output_sub = cli.fm("alarm-summary",
auth_info=subcloud_auth,
fail_ok=False)[1]
central_alarms = table_parser.get_multi_values(
table_parser.table(output_central),
fields=severities,
**{"NAME": subcloud})
subcloud_alarms = table_parser.get_multi_values(
table_parser.table(output_sub), severities)
if central_alarms == subcloud_alarms:
LOG.info(
"'dcmanager alarm summary' output for {} matches 'fm alarm-summary' on "
"{}".format(subcloud, subcloud))
return
time.sleep(30)
assert central_alarms == subcloud_alarms, \
"'dcmanager alarm summary did not match 'fm alarm-summary' on {} " \
"within {}s".format(subcloud, timeout)
def _manage_unmanage_subcloud(subcloud=None,
manage=False,
check_first=True,
fail_ok=False,
con_ssh=None,
auth_info=Tenant.get('admin_platform',
'RegionOne'),
source_openrc=False):
"""
Manage/Unmanage given subcloud(s)
Args:
subcloud:
manage:
check_first:
fail_ok:
Returns:
"""
operation = 'manage' if manage else 'unmanage'
expt_state = '{}d'.format(operation)
if not subcloud:
subcloud = [ProjVar.get_var('PRIMARY_SUBCLOUD')]
elif isinstance(subcloud, str):
subcloud = [subcloud]
subclouds_to_update = list(subcloud)
if check_first:
subclouds_in_state = get_subclouds(mgmt=expt_state,
con_ssh=con_ssh,
auth_info=auth_info)
subclouds_to_update = list(
set(subclouds_to_update) - set(subclouds_in_state))
if not subclouds_to_update:
LOG.info("{} already {}. Do nothing.".format(subcloud, expt_state))
return -1, []
LOG.info("Attempt to {}: {}".format(operation, subclouds_to_update))
failed_subclouds = []
for subcloud_ in subclouds_to_update:
code, out = cli.dcmanager('subcloud ' + operation,
subcloud_,
ssh_client=con_ssh,
fail_ok=True,
auth_info=auth_info,
source_openrc=source_openrc)
if code > 0:
failed_subclouds.append(subcloud_)
if failed_subclouds:
err = "Failed to {} {}".format(operation, failed_subclouds)
if fail_ok:
LOG.info(err)
return 1, failed_subclouds
raise exceptions.DCError(err)
LOG.info(
"Check management status for {} after dcmanager subcloud {}".format(
subclouds_to_update, operation))
mgmt_states = get_subclouds(field='management',
name=subclouds_to_update,
auth_info=auth_info,
con_ssh=con_ssh)
failed_subclouds = [
subclouds_to_update[i] for i in range(len(mgmt_states))
if mgmt_states[i] != expt_state
]
if failed_subclouds:
raise exceptions.DCError("{} not {} after dcmanger subcloud {}".format(
failed_subclouds, expt_state, operation))
return 0, subclouds_to_update
def add_subcloud(subcloud,
subcloud_controller_node,
system_controller_node,
bootstrap_values_path,
deploy_play_book_path,
deploy_values_path,
fail_ok=False,
auth_info=Tenant.get('admin_platform', 'RegionOne'),
source_openrc=None):
"""
"""
operation = 'add'
LOG.info("Attempt to {}: {}".format(operation, subcloud))
if system_controller_node.ssh_conn is None:
msg = 'No ssh connection to System Controller; Cannot add subcloud {} '.format(
subcloud)
LOG.warning(msg)
if fail_ok:
return 1, msg
else:
raise exceptions.DCError(msg)
subcloud_add_config_pathes = [
bootstrap_values_path, deploy_play_book_path, deploy_values_path
]
if not subcloud_controller_node or not bootstrap_values_path or not deploy_play_book_path or not deploy_values_path:
msg = "To add a subcloud all values must be specified"
LOG.warning(msg)
if fail_ok:
return 1, msg
else:
raise exceptions.DCError(msg)
for file_path in subcloud_add_config_pathes:
if system_controller_node.ssh_conn.exec_cmd(
"test -f {}".format(file_path))[0] != 0:
msg = "Subcloud {} is missing config file {} ".format(
subcloud, file_path)
LOG.warning(msg)
if fail_ok:
return 1, msg
else:
raise exceptions.DCError(msg)
args_dict = {
'--bootstrap-address': subcloud_controller_node.host_ip,
'--bootstrap-values': bootstrap_values_path,
'--deploy-playbook': deploy_play_book_path,
'--deploy-values': deploy_values_path,
'--subcloud-password': HostLinuxUser.get_password()
}
opt_args = ''
for key, val in args_dict.items():
if val is not None:
opt_args += '{} {} '.format(key, val)
rc, output = cli.dcmanager('subcloud ' + operation,
opt_args,
ssh_client=system_controller_node.ssh_conn,
fail_ok=fail_ok,
auth_info=auth_info,
source_openrc=source_openrc)
if rc != 0:
msg = "Fail to add subcloud {}: {}".format(subcloud, output)
LOG.warning(msg)
if fail_ok:
return 1, msg
else:
raise exceptions.DCError(msg)
return rc, output
def get_subclouds(field='name',
name=None,
avail=None,
sync=None,
mgmt=None,
deploy=None,
auth_info=Tenant.get('admin_platform', 'RegionOne'),
con_ssh=None,
source_openrc=None,
rtn_dict=False,
evaluate=False,
strict=True,
regex=False,
filter_subclouds=True):
"""
Get subclouds values
Args:
field (str | tuple): fields of value to get
name (str): subcloud name
avail (str): subcloud availability status
sync (str): subcloud sync status
mgmt (str): subcloud management status
deploy (str): subcloud deploy status
auth_info (dict):
con_ssh (SSHClient):
source_openrc (None|bool):
rtn_dict (bool): whether to return dict of field/value pairs
evaluate (bool): whether to convert value to python data type
strict (bool): True to use re.match, False to use re.search
regex (bool): whether to use regex to find value(s)
filter_subclouds (bool): whether to filter out the subclouds that are not in
the --subcloud_list arg
Returns (list | dict):
when rtn_dict is False, list of values
when rtn_dict is True, dict of field/values pairs
"""
table_ = table_parser.table(
cli.dcmanager('subcloud list',
ssh_client=con_ssh,
auth_info=auth_info,
source_openrc=source_openrc)[1])
arg_map = {
'name': name,
'availability': avail,
'sync': sync,
'management': mgmt,
'deploy status': deploy
}
kwargs = {key: val for key, val in arg_map.items() if val}
if filter_subclouds:
filtered_subclouds = table_parser.get_values(table_,
target_header=field,
**kwargs)
subcloud_list = ProjVar.get_var('SUBCLOUD_LIST')
if subcloud_list:
filtered_subclouds = [
subcloud for subcloud in filtered_subclouds
if subcloud in subcloud_list
]
LOG.info('filtered_subclouds: {}'.format(filtered_subclouds))
return filtered_subclouds
else:
return table_parser.get_multi_values(table_,
field,
rtn_dict=rtn_dict,
evaluate=evaluate,
strict=strict,
regex=regex,
**kwargs)
def test_dc_fault_scenario(subcloud_to_test):
"""
Test Fault Scenario on Distributed Cloud
Args:
subcloud_to_test (str): module fixture
Setup:
- Make sure there is consistency between alarm summary on
Central Cloud and on subclouds
Test Steps:
- Make subcloud offline (e. g. delete route)
Step1:
- Ensure suncloud shows offline
Step2:
- Raise alarm on subcloud
- Ensure relative alarm raised on subcloud,
- Ensure system alarm-summary on subcloud has changed
- Ensure dcmanager alarm summary on system controller has no change
Step3:
- Resume connectivity to subcloud (e. g. add route back)
- Ensure suncloud shows online and in-sync
- Ensure system alarm-summary on subcloud matches dcmanager alarm summary on system
controller
Step4:
- Clean alarm on subcloud
- Ensure relative alarm cleared on subcloud
- Ensure system alarm-summary on subcloud matches dcmanager alarm summary on system
controller
"""
ssh_central = ControllerClient.get_active_controller(name="RegionOne")
ssh_subcloud = ControllerClient.get_active_controller(
name=subcloud_to_test)
subcloud_table = {}
try:
code, output = cli.dcmanager(
"subcloud show {}".format(subcloud_to_test),
ssh_client=ssh_central)
gateway = table_parser.get_value_two_col_table(
table_parser.table(output), "management_gateway_ip")
code, hosts_raw = cli.system("host-list", ssh_client=ssh_subcloud)
hosts_id = table_parser.get_values(table_parser.table(hosts_raw), 'id')
for host_id in hosts_id:
code, route_raw = cli.system("host-route-list {}".format(host_id),
ssh_client=ssh_subcloud)
route_table = table_parser.filter_table(
table_parser.table(route_raw), **{'gateway': gateway})
subcloud_table[host_id] = route_table
LOG.tc_step(
"Delete route for subcloud: {} and wait for it to go offline.".
format(subcloud_to_test))
ssh_subcloud = ControllerClient.get_active_controller(
name=subcloud_to_test)
for host_id in subcloud_table:
command = "host-route-delete {}".format(
table_parser.get_values(subcloud_table[host_id], "uuid")[0])
cli.system(command, ssh_client=ssh_subcloud)
dc_helper.wait_for_subcloud_status(subcloud_to_test,
avail=SubcloudStatus.AVAIL_OFFLINE,
timeout=DCTimeout.SYNC,
con_ssh=ssh_central)
LOG.tc_step("Raise alarm on subcloud: {}".format(subcloud_to_test))
ssh_subcloud = ControllerClient.get_active_controller(
name=subcloud_to_test)
code_sub_before, output_sub_before = cli.fm("alarm-summary",
ssh_client=ssh_subcloud)
code_central_before, output_central_before = cli.dcmanager(
'alarm summary')
ssh_subcloud.exec_cmd(
"fmClientCli -c \"### ###300.005###clear###system.vm###host="
"testhost-0### ###critical### ###processing-error###cpu-cycles-limit-exceeded"
"### ###True###True###'\"",
fail_ok=False)
LOG.info("Ensure relative alarm was raised at subcloud: {}".format(
subcloud_to_test))
system_helper.wait_for_alarm(
alarm_id=EventLogID.PROVIDER_NETWORK_FAILURE, con_ssh=ssh_subcloud)
code_sub_after, output_sub_after = cli.fm("alarm-summary",
ssh_client=ssh_subcloud)
code_central_after, output_central_after = cli.dcmanager(
'alarm summary')
LOG.info(
"Ensure fm alarm summary on subcloud: {} has changed but dcmanager alarm"
"summary has not changed".format(subcloud_to_test))
assert output_central_before == output_central_after and output_sub_before != \
output_sub_after
add_routes_to_subcloud(subcloud_to_test, subcloud_table)
dc_helper.wait_for_subcloud_status(subcloud_to_test,
avail=SubcloudStatus.AVAIL_ONLINE,
sync=SubcloudStatus.SYNCED,
timeout=DCTimeout.SYNC,
con_ssh=ssh_central)
alarm_summary_add_and_del(subcloud_to_test)
finally:
cli.dcmanager("subcloud show {}".format(subcloud_to_test),
ssh_client=ssh_central,
fail_ok=True)
add_routes_to_subcloud(subcloud_to_test, subcloud_table, fail_ok=True)
LOG.info("Clear alarm on subcloud: {}".format(subcloud_to_test))
ssh_subcloud.exec_cmd('fmClientCli -D host=testhost-0')
check_alarm_summary_match_subcloud(subcloud=subcloud_to_test)