def teardown():
for dict_ in service_param_restore_dic:
if dict_['action'] == 'delete':
code, msg = system_helper.delete_service_parameter(uuid=dict_['uuid'])
assert code == 0, "Couldn't delete service param {}".format(dict_['uuid'])
elif dict_['action'] == 'modify':
system_helper.modify_service_parameter(service=dict_['service'], section=dict_['section'],
name=dict_['name'], value=dict_['val'], apply=False)
system_helper.apply_service_parameters(service='identity')
def cleanup():
LOG.fixture_step(
"Verifying service parameter {} {} {} is at default value".format(
service, section, name))
res = system_helper.get_service_parameter_values(field='value',
service=service,
section=section,
name=name)[0]
if res != exp_time:
LOG.fixture_step(
"Resetting service parameter {} {} {} to default of {}".format(
service, section, name, exp_time))
system_helper.modify_service_parameter(service,
section,
name,
str(exp_time),
apply=True)
def restore_default_parameters():
LOG.fixture_step(
'Check MNFA service parameter values and revert if needed')
mnfa_threshold_current_val = system_helper.get_service_parameter_values(
service='platform', section='maintenance', name='mnfa_threshold')
mnfa_timeout_default_current_val = system_helper.get_service_parameter_values(
service='platform', section='maintenance', name='mnfa_timeout')
alarms = system_helper.get_alarms(
alarm_id=EventLogID.CONFIG_OUT_OF_DATE)
if alarms or mnfa_threshold_current_val != mnfa_threshold_default_val or mnfa_timeout_default_val != \
mnfa_timeout_default_current_val:
system_helper.modify_service_parameter(
service='platform',
section='maintenance',
name='mnfa_threshold',
apply=False,
value=mnfa_threshold_default_val[0])
system_helper.modify_service_parameter(
service='platform',
check_first=False,
section='maintenance',
name='mnfa_timeout',
apply=True,
value=mnfa_timeout_default_val[0])
def test_multi_node_failure_avoidance(reserve_unreserve_all_hosts_module,
mnfa_timeout, mnfa_threshold):
"""
Test multi node failure avoidance
Args:
mnfa_timeout
mnfa_threshold
reserve_unreserve_all_hosts_module: test fixture to reserve unreserve all vlm nodes for lab under test
Setups:
- Reserve all nodes in vlm
Test Steps:
- Power off compute/storage nodes in vlm using multi-processing to simulate a power outage on computes
- Power on all nodes compute nodes
- Wait for nodes to become degraded state during the mnfa mode
- Wait for nodes to become active state
- Check new event is are created for multi node failure
- Verify the time differences between multi node failure enter and exit in the event log equal to configured
mnfa thereshold value.
"""
hosts_to_check = system_helper.get_hosts(
availability=(HostAvailState.AVAILABLE, HostAvailState.ONLINE))
hosts_to_test = [
host for host in hosts_to_check if 'controller' not in host
]
if len(hosts_to_test) < mnfa_threshold:
skip(
"Compute and storage host count smaller than mnfa threshhold value"
)
elif len(hosts_to_test) > mnfa_threshold + 1:
hosts_to_test = hosts_to_test[:mnfa_threshold + 1]
LOG.info("Online or Available hosts before power-off: {}".format(
hosts_to_check))
start_time = common.get_date_in_format(date_format='%Y-%m-%d %T')
LOG.tc_step('Modify mnfa_timeout parameter to {}'.format(mnfa_timeout))
system_helper.modify_service_parameter(service='platform',
section='maintenance',
name='mnfa_timeout',
apply=True,
value=str(mnfa_timeout))
system_helper.modify_service_parameter(service='platform',
section='maintenance',
name='mnfa_threshold',
apply=True,
value=str(mnfa_threshold))
try:
LOG.tc_step("Power off hosts and check for degraded state: {}".format(
hosts_to_test))
vlm_helper.power_off_hosts_simultaneously(hosts=hosts_to_test)
time.sleep(20)
degraded_hosts = system_helper.get_hosts(
availability=HostAvailState.DEGRADED, hostname=hosts_to_check)
finally:
LOG.tc_step("Power on hosts and ensure they are recovered: {}".format(
hosts_to_test))
vlm_helper.power_on_hosts(hosts=hosts_to_test,
reserve=False,
hosts_to_check=hosts_to_check,
check_interval=20)
assert sorted(degraded_hosts) == sorted(
hosts_to_test), 'Degraded hosts mismatch with powered-off hosts'
LOG.tc_step(
"Check MNFA duration is the same as MNFA timeout value via system event log"
)
active_con = system_helper.get_active_controller_name()
entity_instance_id = 'host={}.event=mnfa_enter'.format(active_con)
first_event = system_helper.wait_for_events(
num=1,
timeout=70,
start=start_time,
fail_ok=True,
strict=False,
event_log_id=EventLogID.MNFA_MODE,
field='Time Stamp',
entity_instance_id=entity_instance_id)
entity_instance_id = 'host={}.event=mnfa_exit'.format(active_con)
second_event = system_helper.wait_for_events(
num=1,
timeout=70,
start=start_time,
fail_ok=False,
strict=False,
event_log_id=EventLogID.MNFA_MODE,
field='Time Stamp',
entity_instance_id=entity_instance_id)
pattern = '%Y-%m-%dT%H:%M:%S'
event_duration = datetime.strptime(second_event[0][:-7],
pattern) - datetime.strptime(
first_event[0][:-7], pattern)
event_duration = event_duration.total_seconds()
assert abs(event_duration - mnfa_timeout) <= 1, 'MNFA event duration {} is different than MNFA timeout value {}'.\
format(event_duration, mnfa_timeout)
def _test_token_expiry(service_params):
"""
Verify that token expiry time can be changed using service parameters
Test Steps:
- Verify that the token expiration is set by default.
- Set token expiry length to set values
- Verify that the length is rejected if it is not between 1 and 4 hours
- Create a token and ensure it expires after the expected expiry time
"""
expire_times = [6000, 7200, 3000, 15500, 3600]
service, section, name = service_params
LOG.tc_step("Verify that token_expiration parameter is defined")
default_exp_time = system_helper.get_service_parameter_values(
field='value', service=service, section=section, name=name)[0]
assert int(
default_exp_time
) == 3600, "Default token_expiration value not 3600, actually {}".format(
default_exp_time)
LOG.tc_step("Verify that tokens now expire after expected time")
token_expire_time = html_helper.get_user_token(field='expires')
expt_time = time.time() + int(default_exp_time)
expt_datetime = datetime.datetime.utcfromtimestamp(expt_time).isoformat()
time_diff = common.get_timedelta_for_isotimes(
expt_datetime, token_expire_time).total_seconds()
LOG.info(
"Expect expiry time to be {}. Token expiry time is {}. Difference is {}."
.format(token_expire_time, expt_datetime, time_diff))
assert -150 < time_diff < 150, "Token expiry time is {}, but token expired {} seconds after expected time.".\
format(expt_time, time_diff)
for expire_time in expire_times:
if expire_time > 14400 or expire_time < 3600:
res, out = system_helper.modify_service_parameter(service,
section,
name,
str(expire_time),
fail_ok=True)
assert 1 == res, "Modifying the expiry time to {} was not rejected".format(
expire_time)
assert "must be between 3600 and 14400 seconds" in out, "Unexpected rejection string"
value = system_helper.get_service_parameter_values(service=service,
section=section,
name=name)
assert expire_time != value, "Expiry time was changed to rejected value"
else:
LOG.tc_step("Set token expiration service parameter to {}".format(
expire_time))
system_helper.modify_service_parameter(service,
section,
name,
str(expire_time),
apply=True)
LOG.tc_step("Verify that tokens now expire after expected time")
token_expire_time = html_helper.get_user_token(field='expires')
expt_time = time.time() + expire_time
expt_datetime = datetime.datetime.utcfromtimestamp(
expt_time).isoformat()
time_diff = common.get_timedelta_for_isotimes(
expt_datetime, token_expire_time).total_seconds()
LOG.info(
"Expect expiry time to be {}. Token expiry time is {}. Difference is {}."
.format(token_expire_time, expt_datetime, time_diff))
assert -150 < time_diff < 150, "Token is not set to expire after {} seconds".format(
expire_time)