def _wait_for_config_apply(auth_info_, con_ssh_=None):
controllers = system_helper.get_controllers(auth_info=auth_info_,
con_ssh=con_ssh_)
for controller in controllers:
system_helper.wait_for_events(
start=start_time,
fail_ok=False,
timeout=60,
entity_instance_id='host=controller',
event_log_id=EventLogID.CONFIG_OUT_OF_DATE,
auth_info=auth_info_,
con_ssh=con_ssh_,
**{
'Entity Instance ID': 'host={}'.format(controller),
'State': 'set'
})
# Extend timeout for controller-1 config-out-date clear to 5min due to CGTS-8497
system_helper.wait_for_events(
start=start_time,
fail_ok=False,
timeout=300,
entity_instance_id='host=controller',
event_log_id=EventLogID.CONFIG_OUT_OF_DATE,
auth_info=auth_info_,
con_ssh=con_ssh_,
**{
'Entity Instance ID': 'host={}'.format(controller),
'State': 'clear'
})
def _test_dynamic_library_change_via_ld_preload_envvar_assignment():
"""
This test attempts to execute a signed/protected binary by pointing to a
modified library via the LD_PRELOAD environment variable. This should not
result in an IMA violation.
Test Steps:
- Execute a signed/protected library binary via the LD_PRELOAD environment
variable.
- Confirm there is no IMA violation.
This maps to TC_17664/T_16353 from US105523 (Dynamic library change via
LD_PRELOAD assignment)
This test also covers TC_17665/T_16397 from US105523 (FM Event Log Updates)
"""
con_ssh = ControllerClient.get_active_controller()
start_time = common.get_date_in_format()
LOG.info("Make a copy of a library used by 'ls'")
source_file = "/lib64/ld-linux-x86-64.so.2"
dest_file = "/lib64/temp.so"
copy_file(source_file, dest_file, cleanup='dest')
ls_cmd = "/usr/bin/ls"
LOG.info("Execute signed binary via LD_PRELOAD")
con_ssh.exec_sudo_cmd("LD_PRELOAD={} ldd {}".format(dest_file, ls_cmd))
# Getting seg fault
LOG.info("Check for IMA event")
events_found = system_helper.wait_for_events(start=start_time,
timeout=60,
num=10,
event_log_id=EventLogID.IMA,
state='log',
severity='major',
fail_ok=True,
strict=False)
assert events_found == [], "Unexpected IMA event found"
def test_ima_event_generation(operation, file_path):
"""
Following IMA violation scenarios are covered:
- append/edit data to/of a monitored file, result in changing of the
hash
- dynamic library changes
- create and execute a files as sysadmin
Test Steps:
- Perform specified file operations
- Check IMA violation event is logged
"""
global files_to_delete
con_ssh = ControllerClient.get_active_controller()
start_time = common.get_date_in_format()
source_file = file_path
backup_file = None
if operation in ('edit_and_execute', 'append_and_execute'):
dest_file = "/usr/sbin/TEMP"
copy_file(source_file, dest_file, cleanup='dest')
if operation == 'edit_and_execute':
LOG.tc_step("Open copy of monitored file and save")
cmd = "vim {} '+:wq!'".format(dest_file)
con_ssh.exec_sudo_cmd(cmd, fail_ok=False)
execute_cmd = "{} -p".format(dest_file)
else:
LOG.tc_step("Append to copy of monitored file")
cmd = 'echo "output" | sudo -S tee -a /usr/sbin/TEMP'.format(
HostLinuxUser.get_password())
con_ssh.exec_cmd(cmd, fail_ok=False)
LOG.tc_step("Execute modified file")
con_ssh.exec_sudo_cmd(dest_file)
execute_cmd = "{}".format(dest_file)
LOG.tc_step("Execute modified file")
con_ssh.exec_sudo_cmd(execute_cmd)
elif operation == 'replace_library':
backup_file = "/root/{}".format(source_file.split('/')[-1])
dest_file_nocsum = "/root/TEMP"
LOG.info("Backup source file {} to {}".format(source_file,
backup_file))
copy_file(source_file, backup_file)
LOG.info("Copy the library without the checksum")
copy_file(source_file, dest_file_nocsum, preserve=False)
LOG.info("Replace the library with the unsigned one")
move_file(dest_file_nocsum, source_file)
elif operation == 'create_and_execute':
dest_file = "{}/TEMP".format(HostLinuxUser.get_home())
create_and_execute(file_path=dest_file, sudo=True)
LOG.tc_step("Check for IMA event")
ima_events = system_helper.wait_for_events(start=start_time,
timeout=60,
num=10,
event_log_id=EventLogID.IMA,
state='log',
severity='major',
fail_ok=True,
strict=False)
if backup_file:
LOG.info("Restore backup file {} to {}".format(backup_file,
source_file))
move_file(backup_file, source_file)
assert ima_events, "IMA event is not generated after {} on " \
"{}".format(operation, file_path)
def test_ima_no_event(operation, file_path):
"""
This test validates following scenarios will not generate IMA event:
- create symlink of a monitored file
- copy a root file with the proper IMA signature, the nexcute it
- make file attribute changes, include: chgrp, chown, chmod
- create and execute a files as sysadmin
Test Steps:
- Perform specified operation on given file
- Confirm IMA violation event is not triggered
Teardown:
- Delete created test file
Maps to TC_17684/TC_17644/TC_17640/TC_17902 from US105523
This test also covers TC_17665/T_16397 from US105523 (FM Event Log Updates)
"""
global files_to_delete
start_time = common.get_date_in_format()
source_file = file_path
con_ssh = ControllerClient.get_active_controller()
LOG.tc_step("{} for {}".format(operation, source_file))
if operation == 'create_symlink':
dest_file = "my_symlink"
create_symlink(source_file, dest_file)
files_to_delete.append(dest_file)
checksum_match = checksum_compare(source_file, dest_file)
assert checksum_match, "SHA256 checksum should match source file and " \
"the symlink but didn't"
elif operation == 'copy_and_execute':
dest_file = "/usr/sbin/TEMP"
copy_file(source_file, dest_file)
files_to_delete.append(dest_file)
LOG.info("Execute the copied file")
con_ssh.exec_sudo_cmd("{} -p".format(dest_file))
elif operation == 'change_file_attributes':
if HostLinuxUser.get_home() != 'sysadmin':
skip('sysadmin user is required to run this test')
dest_file = "/usr/sbin/TEMP"
copy_file(source_file, dest_file)
files_to_delete.append(dest_file)
LOG.info("Change permission of copy")
chmod_file(dest_file, "777")
LOG.info("Changing group ownership of file")
chgrp_file(dest_file, "sys_protected")
LOG.info("Changing file ownership")
chown_file(dest_file, "sysadmin:sys_protected")
elif operation == 'create_and_execute':
dest_file = "{}/TEMP".format(HostLinuxUser.get_home())
create_and_execute(file_path=dest_file, sudo=False)
LOG.tc_step("Ensure no IMA events are raised")
events_found = system_helper.wait_for_events(start=start_time,
timeout=60,
num=10,
event_log_id=EventLogID.IMA,
fail_ok=True,
strict=False)
assert not events_found, "Unexpected IMA events found"
def test_transition_sensorgroup_actions(host,
event_type,
action_level,
action,
suppression,
expt_alarm,
expt_host_avail,
new_action,
new_suppression,
new_expt_alarm,
new_expt_host_avail,
sensor_data_fit):
"""
Verify the sensorgroup can properly transition from one action to another when
an event remains unchanged.
Test Steps:
- Get a sensorgroup to test
- Set the event level and expected action
- trigger an out-of-scope event for that sensorgroup
- verify that the expected action is taken
- transition the sensorgroup action
- verify the new action is taken
"""
bmc_hosts = sensor_data_fit
if host not in bmc_hosts:
skip("{} is not configured with BMC sensor".format(host))
global HOST
HOST = host
# Get a sensor to validate
expt_severity = action_level.split('_')[-1] if 'yes' in expt_alarm else None
new_expt_severity = action_level.split('_')[-1] if 'yes' in new_expt_alarm else None
if suppression is not None:
suppression = True if suppression == 'suppressed' else False
if new_suppression is not None:
new_suppression = True if new_suppression == 'suppressed' else False
for sensorgroup_name in bmc_helper.get_sensorgroup_name(host):
LOG.tc_step("Validating that sensorgroup: {} can be set to sensor action: {} for event level: {}".
format(sensorgroup_name, action, action_level))
# Set the sensorgroup action, suppress state, and audit interval
bmc_helper.modify_sensorgroup(host, sensorgroup_name, value='name', audit_interval=10, suppress=suppression,
**{action_level: action})
# Get a sensor that is part of the sensorgroup
sensor_name = bmc_helper.get_first_sensor_from_sensorgroup(sensorgroup_name, host)
entity_id = 'host={}.sensor={}'.format(host, sensor_name)
LOG.tc_step("Trigger event for sensorgroup: {} and sensor name: {}".format(sensorgroup_name, sensor_name))
bmc_helper.trigger_event(host, sensor_name, event_type)
LOG.tc_step("Check the alarm status for sensor: {}".format(sensor_name))
res = system_helper.wait_for_alarm(alarm_id=EventLogID.BMC_SENSOR_ACTION, timeout=60, entity_id=entity_id,
severity=expt_severity, strict=False, fail_ok=True)[0]
if expt_alarm == 'yes_alarm':
assert res, "FAIL: Alarm expected but no alarms found for sensor on {}".format(host)
else:
assert not res, "FAIL: Alarm raised but no alarms were expected for sensor on {}".format(host)
LOG.tc_step("Check the host status for sensor: {}".format(sensor_name))
system_helper.wait_for_host_values(host, timeout=90, availability=expt_host_avail, fail_ok=False)
start_time = common.get_date_in_format()
# modify sensorgroup with new action/suppression level
LOG.tc_step("Transition sensorgroup: {} from current sensor action: {} to new sensor action: {} "
"for event level: {}".format(sensorgroup_name, action, new_action, action_level))
bmc_helper.modify_sensorgroup(host, sensorgroup_name, value='name', suppress=new_suppression,
**{action_level: new_action})
# Verify the new action is taken
LOG.tc_step("Check alarm status after transition from {} to {} for {}".format(action, new_action, sensor_name))
if new_expt_alarm == 'yes_alarm':
system_helper.wait_for_alarm(alarm_id=EventLogID.BMC_SENSOR_ACTION, entity_id=entity_id,
severity=new_expt_severity, timeout=60, strict=False, fail_ok=False)
else:
events = system_helper.wait_for_events(timeout=60, num=10, event_log_id=EventLogID.BMC_SENSOR_ACTION,
entity_instance_id=entity_id, start=start_time, state='log',
fail_ok=True, strict=False, severity=new_expt_severity)
if new_expt_alarm == 'yes_log':
assert events, "No event log found for {} {} {} event".format(host, sensorgroup_name, action_level)
else:
assert not events, "Event logged unexpectedly for sensor on {}".format(host)
system_helper.wait_for_alarm_gone(EventLogID.BMC_SENSOR_ACTION, entity_id=entity_id, strict=False,
timeout=5, fail_ok=False)
LOG.tc_step("Check the host status for sensor: {}".format(sensor_name))
system_helper.wait_for_host_values(host, timeout=90, availability=new_expt_host_avail, fail_ok=False)
LOG.tc_step("Check the alarm clears and host in available state after clearing events")
bmc_helper.clear_events(host)
system_helper.wait_for_alarm_gone(alarm_id=EventLogID.BMC_SENSOR_ACTION, entity_id=host, strict=False,
timeout=60)
system_helper.wait_for_host_values(host, fail_ok=False, availability='available')
HOST = ''
def test_sensorgroup_power_cycle(host,
eventlevel,
action,
expected_host_state,
expected_alarm_state,
event_type,
suppressionlevel, sensor_data_fit):
"""
Verify that the sensorgroup action taken for an event is valid.
Test Steps:
- Get a sensorgroup to test
- Set the event level and expected action
- trigger an out-of-scope event for that sensorgroup
- verify that the expected action is taken
"""
bmc_hosts = sensor_data_fit
if host not in bmc_hosts:
skip("{} is not configured with BMC sensor".format(host))
global HOST
HOST = host
if suppressionlevel == 'suppressed':
# global SUPPRESSED
# SUPPRESSED = host
suppress = True
else:
suppress = False
expt_severity = eventlevel.split('_')[-1] if 'yes' in expected_alarm_state else None
# Get a sensor to validate
sensorgroup_name = random.choice(bmc_helper.get_sensor_names(host, sensor_group=True))
for i in range(4):
LOG.info("################## iter {} #########################".format(i+1))
LOG.tc_step("Validating that sensorgroup: {} "
"can be set to sensor action: {} "
"for event level: {}".format(sensorgroup_name, action,
eventlevel))
# Set the event level and action
bmc_helper.modify_sensorgroup(host, sensorgroup_name, value='name', suppress=suppress, audit_interval=10,
**{eventlevel: action})
# Get a sensor that is part of the sensorgroup
sensor_name = bmc_helper.get_first_sensor_from_sensorgroup(sensorgroup_name, host)
entity_id = 'host={}.sensor={}'.format(host, sensor_name)
LOG.tc_step("Trigger event for sensorgroup: {} and sensor name: {}".
format(sensorgroup_name, sensor_name))
if action in ['power-cycle', 'reset']:
HostsToRecover.add(host)
start_time = common.get_date_in_format()
bmc_helper.trigger_event(host, sensor_name, event_type)
LOG.tc_step("Check sensor status and alarm for {}".format(sensor_name))
if expected_alarm_state == 'yes_alarm':
system_helper.wait_for_alarm(alarm_id=EventLogID.BMC_SENSOR_ACTION, entity_id=entity_id,
severity=expt_severity, timeout=60, strict=False, fail_ok=False)
else:
events = system_helper.wait_for_events(timeout=60, num=10, event_log_id=EventLogID.BMC_SENSOR_ACTION,
entity_instance_id=entity_id, start=start_time, state='log',
severity=expt_severity, fail_ok=True, strict=False)
if expected_alarm_state == 'yes_log':
assert events, "No event log found for {} {} {} event".format(host, sensorgroup_name, eventlevel)
else:
assert not events, "Event logged unexpectedly for sensor on {}".format(host)
system_helper.wait_for_alarm_gone(EventLogID.BMC_SENSOR_ACTION, entity_id=entity_id, strict=False,
timeout=5, fail_ok=False)
LOG.tc_step("Check the host status for sensor: {}".format(sensor_name))
host_state_timeout = 120
if action == 'reset':
host_state_timeout = 1080 # 15 min reset interval in between two reset triggers
system_helper.wait_for_host_values(host, timeout=host_state_timeout, fail_ok=False,
availability=expected_host_state)
if action == 'power-cycle':
system_helper.wait_for_host_values(host, timeout=20, task=HostTask.POWER_CYCLE, strict=False)
LOG.tc_step("Check the alarm clears and host in available state after clearing events")
bmc_helper.clear_events(host)
system_helper.wait_for_alarm_gone(alarm_id=EventLogID.BMC_SENSOR_ACTION, entity_id=host, strict=False,
timeout=60)
wait_time = 3000 if action == 'power-cycle' else HostTimeout.REBOOT
expt_states = {'availability': 'available'}
strict = True
if action == 'power-cycle' and i == 3:
wait_time = 1200
strict = False
expt_states = {'availability': HostAvailState.POWER_OFF,
'operational': HostOperState.DISABLED,
'administrative': HostAdminState.UNLOCKED,
'task': HostTask.POWER_DOWN}
system_helper.wait_for_host_values(host, fail_ok=False, timeout=wait_time, strict=strict, **expt_states)
LOG.tc_step("Power on {} after test ends".format(host))
host_helper.lock_host(host=host)
host_helper.power_on_host(host=host)
HOST = ''
def test_resize_drbd_filesystem_while_resize_inprogress():
"""
This test attempts to resize a drbd filesystem while an existing drbd
resize is in progress. This should be rejected.
Arguments:
- None
Test steps:
1. Increase the size of backup to allow for test to proceed.
2. Wait for alarms to clear and then check the underlying filesystem is
updated
2. Attempt to resize the glance filesystem. This should be successful.
3. Attempt to resize cgcs again immediately. This should be rejected.
Assumptions:
- None
"""
start_time = common.get_date_in_format()
drbdfs_val = {}
fs = "extension"
LOG.tc_step(
"Increase the {} size before proceeding with rest of test".format(fs))
drbdfs_val[fs] = storage_helper.get_controllerfs_values(fs)[0]
LOG.info("Current value of {} is {}".format(fs, drbdfs_val[fs]))
drbdfs_val[fs] = int(drbdfs_val[fs]) + 5
LOG.info("Will attempt to increase the value of {} to {}".format(
fs, drbdfs_val[fs]))
LOG.tc_step("Increase the size of filesystems")
storage_helper.modify_controllerfs(**drbdfs_val)
hosts = system_helper.get_controllers()
for host in hosts:
system_helper.wait_for_events(
event_log_id=EventLogID.CONFIG_OUT_OF_DATE,
start=start_time,
entity_instance_id="host={}".format(host),
strict=False,
**{'state': 'set'})
for host in hosts:
system_helper.wait_for_alarm_gone(
alarm_id=EventLogID.CONFIG_OUT_OF_DATE,
entity_id="host={}".format(host),
timeout=600)
LOG.tc_step(
"Confirm the underlying filesystem size matches what is expected")
storage_helper.check_controllerfs(**drbdfs_val)
drbdfs_val = {}
fs = "database"
LOG.tc_step("Determine the current filesystem size")
value = storage_helper.get_controllerfs_values(fs)[0]
LOG.info("Current value of {} is {}".format(fs, value))
drbdfs_val[fs] = int(value) + 1
LOG.info("Will attempt to increase the value of {} to {}".format(
fs, drbdfs_val[fs]))
LOG.tc_step("Increase the size of filesystems")
storage_helper.modify_controllerfs(**drbdfs_val)
LOG.tc_step("Attempt to increase the size of the filesystem again")
drbdfs_val[fs] = int(drbdfs_val[fs]) + 1
code = storage_helper.modify_controllerfs(fail_ok=True, **drbdfs_val)[0]
assert 1 == code, "Filesystem modify succeeded while failure is expected: {}".format(
drbdfs_val)
# Appearance of sync alarm is delayed so wait for it to appear and then
# clear
if not system_helper.is_aio_simplex():
system_helper.wait_for_alarm(alarm_id=EventLogID.CON_DRBD_SYNC,
timeout=300)
system_helper.wait_for_alarm_gone(alarm_id=EventLogID.CON_DRBD_SYNC,
timeout=300)
def test_system_alarms_and_events_on_lock_unlock_compute(no_simplex):
"""
Verify fm alarm-show command
Test Steps:
- Delete active alarms
- Lock a host
- Check active alarm generated for host lock
- Check relative values are the same in fm alarm-list and fm alarm-show
<uuid>
- Check host lock 'set' event logged via fm event-list
- Unlock host
- Check active alarms cleared via fm alarm-list
- Check host lock 'clear' event logged via fm event-list
"""
# Remove following step because it's unnecessary and fails the test when
# alarm is re-generated
# # Clear the alarms currently present
# LOG.tc_step("Clear the alarms table")
# system_helper.delete_alarms()
# Raise a new alarm by locking a compute node
# Get the compute
compute_host = host_helper.get_up_hypervisors()[0]
if compute_host == system_helper.get_active_controller_name():
compute_host = system_helper.get_standby_controller_name()
if not compute_host:
skip('Standby controller unavailable')
LOG.tc_step("Lock a nova hypervisor host {}".format(compute_host))
pre_lock_time = common.get_date_in_format()
HostsToRecover.add(compute_host)
host_helper.lock_host(compute_host)
LOG.tc_step("Check host lock alarm is generated")
post_lock_alarms = \
system_helper.wait_for_alarm(field='UUID', entity_id=compute_host,
reason=compute_host,
alarm_id=EventLogID.HOST_LOCK,
strict=False,
fail_ok=False)[1]
LOG.tc_step(
"Check related fields in fm alarm-list and fm alarm-show are of the "
"same values")
post_lock_alarms_tab = system_helper.get_alarms_table(uuid=True)
alarms_l = ['Alarm ID', 'Entity ID', 'Severity', 'Reason Text']
alarms_s = ['alarm_id', 'entity_instance_id', 'severity', 'reason_text']
# Only 1 alarm since we are now checking the specific alarm ID
for post_alarm in post_lock_alarms:
LOG.tc_step(
"Verify {} for alarm {} in alarm-list are in sync with "
"alarm-show".format(
alarms_l, post_alarm))
alarm_show_tab = table_parser.table(cli.fm('alarm-show', post_alarm)[1])
alarm_list_tab = table_parser.filter_table(post_lock_alarms_tab,
UUID=post_alarm)
for i in range(len(alarms_l)):
alarm_l_val = table_parser.get_column(alarm_list_tab,
alarms_l[i])[0]
alarm_s_val = table_parser.get_value_two_col_table(alarm_show_tab,
alarms_s[i])
assert alarm_l_val == alarm_s_val, \
"{} value in alarm-list: {} is different than alarm-show: " \
"{}".format(alarms_l[i], alarm_l_val, alarm_s_val)
LOG.tc_step("Check host lock is logged via fm event-list")
system_helper.wait_for_events(entity_instance_id=compute_host,
start=pre_lock_time, timeout=60,
event_log_id=EventLogID.HOST_LOCK,
fail_ok=False, **{'state': 'set'})
pre_unlock_time = common.get_date_in_format()
LOG.tc_step("Unlock {}".format(compute_host))
host_helper.unlock_host(compute_host)
LOG.tc_step("Check host lock active alarm cleared")
alarm_sets = [(EventLogID.HOST_LOCK, compute_host)]
system_helper.wait_for_alarms_gone(alarm_sets, fail_ok=False)
LOG.tc_step("Check host lock clear event logged")
system_helper.wait_for_events(event_log_id=EventLogID.HOST_LOCK,
start=pre_unlock_time,
entity_instance_id=compute_host,
fail_ok=False, **{'state': 'clear'})
def test_multi_node_failure_avoidance(reserve_unreserve_all_hosts_module,
mnfa_timeout, mnfa_threshold):
"""
Test multi node failure avoidance
Args:
mnfa_timeout
mnfa_threshold
reserve_unreserve_all_hosts_module: test fixture to reserve unreserve all vlm nodes for lab under test
Setups:
- Reserve all nodes in vlm
Test Steps:
- Power off compute/storage nodes in vlm using multi-processing to simulate a power outage on computes
- Power on all nodes compute nodes
- Wait for nodes to become degraded state during the mnfa mode
- Wait for nodes to become active state
- Check new event is are created for multi node failure
- Verify the time differences between multi node failure enter and exit in the event log equal to configured
mnfa thereshold value.
"""
hosts_to_check = system_helper.get_hosts(
availability=(HostAvailState.AVAILABLE, HostAvailState.ONLINE))
hosts_to_test = [
host for host in hosts_to_check if 'controller' not in host
]
if len(hosts_to_test) < mnfa_threshold:
skip(
"Compute and storage host count smaller than mnfa threshhold value"
)
elif len(hosts_to_test) > mnfa_threshold + 1:
hosts_to_test = hosts_to_test[:mnfa_threshold + 1]
LOG.info("Online or Available hosts before power-off: {}".format(
hosts_to_check))
start_time = common.get_date_in_format(date_format='%Y-%m-%d %T')
LOG.tc_step('Modify mnfa_timeout parameter to {}'.format(mnfa_timeout))
system_helper.modify_service_parameter(service='platform',
section='maintenance',
name='mnfa_timeout',
apply=True,
value=str(mnfa_timeout))
system_helper.modify_service_parameter(service='platform',
section='maintenance',
name='mnfa_threshold',
apply=True,
value=str(mnfa_threshold))
try:
LOG.tc_step("Power off hosts and check for degraded state: {}".format(
hosts_to_test))
vlm_helper.power_off_hosts_simultaneously(hosts=hosts_to_test)
time.sleep(20)
degraded_hosts = system_helper.get_hosts(
availability=HostAvailState.DEGRADED, hostname=hosts_to_check)
finally:
LOG.tc_step("Power on hosts and ensure they are recovered: {}".format(
hosts_to_test))
vlm_helper.power_on_hosts(hosts=hosts_to_test,
reserve=False,
hosts_to_check=hosts_to_check,
check_interval=20)
assert sorted(degraded_hosts) == sorted(
hosts_to_test), 'Degraded hosts mismatch with powered-off hosts'
LOG.tc_step(
"Check MNFA duration is the same as MNFA timeout value via system event log"
)
active_con = system_helper.get_active_controller_name()
entity_instance_id = 'host={}.event=mnfa_enter'.format(active_con)
first_event = system_helper.wait_for_events(
num=1,
timeout=70,
start=start_time,
fail_ok=True,
strict=False,
event_log_id=EventLogID.MNFA_MODE,
field='Time Stamp',
entity_instance_id=entity_instance_id)
entity_instance_id = 'host={}.event=mnfa_exit'.format(active_con)
second_event = system_helper.wait_for_events(
num=1,
timeout=70,
start=start_time,
fail_ok=False,
strict=False,
event_log_id=EventLogID.MNFA_MODE,
field='Time Stamp',
entity_instance_id=entity_instance_id)
pattern = '%Y-%m-%dT%H:%M:%S'
event_duration = datetime.strptime(second_event[0][:-7],
pattern) - datetime.strptime(
first_event[0][:-7], pattern)
event_duration = event_duration.total_seconds()
assert abs(event_duration - mnfa_timeout) <= 1, 'MNFA event duration {} is different than MNFA timeout value {}'.\
format(event_duration, mnfa_timeout)