def adaptive_recover(dut, localhost, fanouthosts, check_results, wait_time): outstanding_action = None for result in check_results: if result['failed']: if result['check_item'] == 'interfaces': action = __recover_interfaces(dut, fanouthosts, result, wait_time) elif result['check_item'] == 'services': action = __recover_services(dut, result) elif result['check_item'] in ['processes', 'bgp']: action = 'config_reload' else: action = 'reboot' # Any action can override no action or 'config_reload'. # 'reboot' is last resort and cannot be overridden. if action and (not outstanding_action or outstanding_action == 'config_reload'): outstanding_action = action logging.warning( "Restoring {} with proposed action: {}, final action: {}". format(result, action, outstanding_action)) if outstanding_action: if outstanding_action == "config_reload" and config_force_option_supported( dut): outstanding_action = "config_reload_f" method = constants.RECOVER_METHODS[outstanding_action] wait_time = method['recover_wait'] if method["reboot"]: reboot_dut(dut, localhost, method["cmd"]) else: __recover_with_command(dut, method['cmd'], wait_time)
def recover(dut, localhost, fanouthosts, check_results, recover_method): logger.warning("Try to recover %s using method %s" % (dut.hostname, recover_method)) if recover_method == "config_reload" and config_force_option_supported(dut): recover_method = "config_reload_f" method = constants.RECOVER_METHODS[recover_method] wait_time = method['recover_wait'] if method["adaptive"]: adaptive_recover(dut, localhost, fanouthosts, check_results, wait_time) elif method["reboot"]: reboot_dut(dut, localhost, method["cmd"], wait_time) else: __recover_with_command(dut, method['cmd'], wait_time)
def test_reload_configuration(duthosts, rand_one_dut_hostname, conn_graph_facts, xcvr_skip_list): """ @summary: This test case is to reload the configuration and check platform status """ duthost = duthosts[rand_one_dut_hostname] interfaces = conn_graph_facts["device_conn"][duthost.hostname] asic_type = duthost.facts["asic_type"] if config_force_option_supported(duthost): assert wait_until(300, 20, 0, config_system_checks_passed, duthost) logging.info("Reload configuration") duthost.shell("sudo config reload -y &>/dev/null", executable="/bin/bash") logging.info("Wait until all critical services are fully started") wait_critical_processes(duthost) logging.info("Wait some time for all the transceivers to be detected") assert wait_until(300, 20, 0, check_all_interface_information, duthost, interfaces, xcvr_skip_list), \ "Not all transceivers are detected in 300 seconds" logging.info("Check transceiver status") for asic_index in duthost.get_frontend_asic_ids(): # Get the interfaces pertaining to that asic interface_list = get_port_map(duthost, asic_index) interfaces_per_asic = { k: v for k, v in interface_list.items() if k in interfaces } check_transceiver_basic(duthost, asic_index, interfaces_per_asic, xcvr_skip_list) if asic_type in ["mellanox"]: from .mellanox.check_hw_mgmt_service import check_hw_management_service from .mellanox.check_sysfs import check_sysfs logging.info("Check the hw-management service") check_hw_management_service(duthost) logging.info("Check sysfs") check_sysfs(duthost)
def test_reload_configuration_checks(duthosts, rand_one_dut_hostname, localhost, conn_graph_facts, xcvr_skip_list): """ @summary: This test case is to test various system checks in config reload """ duthost = duthosts[rand_one_dut_hostname] if not config_force_option_supported(duthost): return reboot(duthost, localhost, reboot_type="cold", wait=5) logging.info("Reload configuration check") out = duthost.shell("sudo config reload -y", executable="/bin/bash") # config reload command shouldn't work immediately after system reboot assert "Retry later" in out['stdout'] assert wait_until(300, 20, 0, config_system_checks_passed, duthost) # After the system checks succeed the config reload command should not throw error out = duthost.shell("sudo config reload -y", executable="/bin/bash") assert "Retry later" not in out['stdout'] # Immediately after one config reload command, another shouldn't execute and wait for system checks logging.info("Checking config reload after system is up") out = duthost.shell("sudo config reload -y", executable="/bin/bash") assert "Retry later" in out['stdout'] assert wait_until(300, 20, 0, config_system_checks_passed, duthost) logging.info("Stopping swss docker and checking config reload") duthost.shell("sudo service swss stop") # Without swss running config reload option should not proceed out = duthost.shell("sudo config reload -y", executable="/bin/bash") assert "Retry later" in out['stdout'] # However with force option config reload should proceed logging.info("Performing force config reload") out = duthost.shell("sudo config reload -y -f", executable="/bin/bash") assert "Retry later" not in out['stdout'] assert wait_until(300, 20, 0, config_system_checks_passed, duthost)