def adaptive_recover(dut, localhost, fanouthosts, check_results, wait_time):
outstanding_action = None
for result in check_results:
if result['failed']:
if result['check_item'] == 'interfaces':
action = __recover_interfaces(dut, fanouthosts, result,
wait_time)
elif result['check_item'] == 'services':
action = __recover_services(dut, result)
elif result['check_item'] in ['processes', 'bgp']:
action = 'config_reload'
else:
action = 'reboot'
# Any action can override no action or 'config_reload'.
# 'reboot' is last resort and cannot be overridden.
if action and (not outstanding_action
or outstanding_action == 'config_reload'):
outstanding_action = action
logging.warning(
"Restoring {} with proposed action: {}, final action: {}".
format(result, action, outstanding_action))
if outstanding_action:
if outstanding_action == "config_reload" and config_force_option_supported(
dut):
outstanding_action = "config_reload_f"
method = constants.RECOVER_METHODS[outstanding_action]
wait_time = method['recover_wait']
if method["reboot"]:
reboot_dut(dut, localhost, method["cmd"])
else:
__recover_with_command(dut, method['cmd'], wait_time)
def recover(dut, localhost, fanouthosts, check_results, recover_method):
logger.warning("Try to recover %s using method %s" % (dut.hostname, recover_method))
if recover_method == "config_reload" and config_force_option_supported(dut):
recover_method = "config_reload_f"
method = constants.RECOVER_METHODS[recover_method]
wait_time = method['recover_wait']
if method["adaptive"]:
adaptive_recover(dut, localhost, fanouthosts, check_results, wait_time)
elif method["reboot"]:
reboot_dut(dut, localhost, method["cmd"], wait_time)
else:
__recover_with_command(dut, method['cmd'], wait_time)
def test_reload_configuration(duthosts, rand_one_dut_hostname,
conn_graph_facts, xcvr_skip_list):
"""
@summary: This test case is to reload the configuration and check platform status
"""
duthost = duthosts[rand_one_dut_hostname]
interfaces = conn_graph_facts["device_conn"][duthost.hostname]
asic_type = duthost.facts["asic_type"]
if config_force_option_supported(duthost):
assert wait_until(300, 20, 0, config_system_checks_passed, duthost)
logging.info("Reload configuration")
duthost.shell("sudo config reload -y &>/dev/null", executable="/bin/bash")
logging.info("Wait until all critical services are fully started")
wait_critical_processes(duthost)
logging.info("Wait some time for all the transceivers to be detected")
assert wait_until(300, 20, 0, check_all_interface_information, duthost, interfaces, xcvr_skip_list), \
"Not all transceivers are detected in 300 seconds"
logging.info("Check transceiver status")
for asic_index in duthost.get_frontend_asic_ids():
# Get the interfaces pertaining to that asic
interface_list = get_port_map(duthost, asic_index)
interfaces_per_asic = {
k: v
for k, v in interface_list.items() if k in interfaces
}
check_transceiver_basic(duthost, asic_index, interfaces_per_asic,
xcvr_skip_list)
if asic_type in ["mellanox"]:
from .mellanox.check_hw_mgmt_service import check_hw_management_service
from .mellanox.check_sysfs import check_sysfs
logging.info("Check the hw-management service")
check_hw_management_service(duthost)
logging.info("Check sysfs")
check_sysfs(duthost)
def test_reload_configuration_checks(duthosts, rand_one_dut_hostname,
localhost, conn_graph_facts,
xcvr_skip_list):
"""
@summary: This test case is to test various system checks in config reload
"""
duthost = duthosts[rand_one_dut_hostname]
if not config_force_option_supported(duthost):
return
reboot(duthost, localhost, reboot_type="cold", wait=5)
logging.info("Reload configuration check")
out = duthost.shell("sudo config reload -y", executable="/bin/bash")
# config reload command shouldn't work immediately after system reboot
assert "Retry later" in out['stdout']
assert wait_until(300, 20, 0, config_system_checks_passed, duthost)
# After the system checks succeed the config reload command should not throw error
out = duthost.shell("sudo config reload -y", executable="/bin/bash")
assert "Retry later" not in out['stdout']
# Immediately after one config reload command, another shouldn't execute and wait for system checks
logging.info("Checking config reload after system is up")
out = duthost.shell("sudo config reload -y", executable="/bin/bash")
assert "Retry later" in out['stdout']
assert wait_until(300, 20, 0, config_system_checks_passed, duthost)
logging.info("Stopping swss docker and checking config reload")
duthost.shell("sudo service swss stop")
# Without swss running config reload option should not proceed
out = duthost.shell("sudo config reload -y", executable="/bin/bash")
assert "Retry later" in out['stdout']
# However with force option config reload should proceed
logging.info("Performing force config reload")
out = duthost.shell("sudo config reload -y -f", executable="/bin/bash")
assert "Retry later" not in out['stdout']
assert wait_until(300, 20, 0, config_system_checks_passed, duthost)