def restart_service_and_check(localhost, dut, service, interfaces): """ Restart specified service and check platform status """ logging.info("Restart the %s service" % service) dut.command("sudo systemctl restart %s" % service) logging.info("Wait until all critical services are fully started") check_critical_services(dut) logging.info("Wait some time for all the transceivers to be detected") pytest_assert(wait_until(300, 20, check_interface_information, dut, interfaces), "Not all interface information are detected within 300 seconds") logging.info("Check transceiver status") check_transceiver_basic(dut, interfaces) if dut.facts["asic_type"] in ["mellanox"]: from .mellanox.check_hw_mgmt_service import check_hw_management_service from .mellanox.check_sysfs import check_sysfs logging.info("Check the hw-management service") check_hw_management_service(dut) logging.info("Check sysfs") check_sysfs(dut) logging.info("Check that critical processes are healthy for 60 seconds") check_critical_processes(dut, 60)
def restart_service_and_check(localhost, dut, service, interfaces): """ Restart specified service and check platform status """ logging.info("Restart the %s service" % service) dut.command("sudo systemctl restart %s" % service) logging.info("Wait until all critical services are fully started") check_critical_services(dut) logging.info("Wait some time for all the transceivers to be detected") assert wait_until(300, 20, check_interface_information, dut, interfaces), \ "Not all interface information are detected within 300 seconds" logging.info("Check transceiver status") check_transceiver_basic(dut, interfaces) if dut.facts["asic_type"] in ["mellanox"]: current_file_dir = os.path.dirname(os.path.realpath(__file__)) sub_folder_dir = os.path.join(current_file_dir, "mellanox") if sub_folder_dir not in sys.path: sys.path.append(sub_folder_dir) from check_hw_mgmt_service import check_hw_management_service from check_sysfs import check_sysfs logging.info("Check the hw-management service") check_hw_management_service(dut) logging.info("Check sysfs") check_sysfs(dut)
def test_reload_configuration(duthost, conn_graph_facts): """ @summary: This test case is to reload the configuration and check platform status """ interfaces = conn_graph_facts["device_conn"] asic_type = duthost.facts["asic_type"] logging.info("Reload configuration") duthost.shell("sudo config reload -y &>/dev/null", executable="/bin/bash") logging.info("Wait until all critical services are fully started") check_critical_services(duthost) logging.info("Wait some time for all the transceivers to be detected") assert wait_until(300, 20, check_interface_information, duthost, interfaces), \ "Not all transceivers are detected in 300 seconds" logging.info("Check transceiver status") check_transceiver_basic(duthost, interfaces) if asic_type in ["mellanox"]: from .mellanox.check_hw_mgmt_service import check_hw_management_service from .mellanox.check_sysfs import check_sysfs logging.info("Check the hw-management service") check_hw_management_service(duthost) logging.info("Check sysfs") check_sysfs(duthost)
def test_reload_configuration(testbed_devices, conn_graph_facts): """ @summary: This test case is to reload the configuration and check platform status """ ans_host = testbed_devices["dut"] interfaces = conn_graph_facts["device_conn"] asic_type = ans_host.facts["asic_type"] logging.info("Reload configuration") ans_host.command("sudo config reload -y") logging.info("Wait until all critical services are fully started") check_critical_services(ans_host) logging.info("Wait some time for all the transceivers to be detected") assert wait_until(300, 20, check_interface_information, ans_host, interfaces), \ "Not all transceivers are detected in 300 seconds" logging.info("Check transceiver status") check_transceiver_basic(ans_host, interfaces) if asic_type in ["mellanox"]: current_file_dir = os.path.dirname(os.path.realpath(__file__)) sub_folder_dir = os.path.join(current_file_dir, "mellanox") if sub_folder_dir not in sys.path: sys.path.append(sub_folder_dir) from check_hw_mgmt_service import check_hw_management_service from check_sysfs import check_sysfs logging.info("Check the hw-management service") check_hw_management_service(ans_host) logging.info("Check sysfs") check_sysfs(ans_host)
def reboot_and_check(localhost, dut, interfaces, reboot_type="cold"): """ Perform the specified type of reboot and check platform status. """ logging.info("Run %s reboot on DUT" % reboot_type) if reboot_type == "cold": reboot_cmd = "reboot" reboot_timeout = 300 elif reboot_type == "fast": reboot_cmd = "fast-reboot" reboot_timeout = 180 elif reboot_type == "warm": reboot_cmd = "warm-reboot" reboot_timeout = 180 else: assert False, "Reboot type %s is not supported" % reboot_type process, queue = dut.command(reboot_cmd, module_async=True) logging.info("Wait for DUT to go down") res = localhost.wait_for(host=dut.hostname, port=22, state="stopped", delay=10, timeout=120, module_ignore_errors=True) if "failed" in res: if process.is_alive(): logging.error("Command '%s' is not completed" % reboot_cmd) process.terminate() logging.error("reboot result %s" % str(queue.get())) assert False, "DUT did not go down" logging.info("Wait for DUT to come back") localhost.wait_for(host=dut.hostname, port=22, state="started", delay=10, timeout=reboot_timeout) logging.info("Wait until all critical services are fully started") check_critical_services(dut) logging.info("Wait some time for all the transceivers to be detected") assert wait_until(300, 20, all_transceivers_detected, dut, interfaces), \ "Not all transceivers are detected in 300 seconds" logging.info("Check interface status") check_interface_status(dut, interfaces) logging.info("Check transceiver status") check_transceiver_basic(dut, interfaces) if dut.facts["asic_type"] in ["mellanox"]: current_file_dir = os.path.dirname(os.path.realpath(__file__)) sub_folder_dir = os.path.join(current_file_dir, "mellanox") if sub_folder_dir not in sys.path: sys.path.append(sub_folder_dir) from check_hw_mgmt_service import check_hw_management_service from check_sysfs import check_sysfs logging.info("Check the hw-management service") check_hw_management_service(dut) logging.info("Check sysfs") check_sysfs(dut)
def check_interfaces_and_services(dut, interfaces, reboot_type=None): """ Perform a further check after reboot-cause, including transceiver status, interface status @param localhost: The Localhost object. @param dut: The AnsibleHost object of DUT. @param interfaces: DUT's interfaces defined by minigraph """ logging.info("Wait until all critical services are fully started") check_critical_services(dut) if reboot_type is not None: logging.info("Check reboot cause") reboot_cause = reboot_ctrl_dict[reboot_type]["cause"] assert wait_until(MAX_WAIT_TIME_FOR_REBOOT_CAUSE, 20, check_reboot_cause, dut, reboot_cause), \ "got reboot-cause failed after rebooted by %s" % reboot_cause if reboot_ctrl_dict[reboot_type]["test_reboot_cause_only"]: logging.info( "Further checking skipped for %s test which intends to verify reboot-cause only" % reboot_type) return logging.info("Wait %d seconds for all the transceivers to be detected" % MAX_WAIT_TIME_FOR_INTERFACES) assert wait_until(MAX_WAIT_TIME_FOR_INTERFACES, 20, check_interface_information, dut, interfaces), \ "Not all transceivers are detected or interfaces are up in %d seconds" % MAX_WAIT_TIME_FOR_INTERFACES logging.info("Check transceiver status") check_transceiver_basic(dut, interfaces) logging.info("Check pmon daemon status") assert check_pmon_daemon_status(dut), "Not all pmon daemons running." if dut.facts["asic_type"] in ["mellanox"]: current_file_dir = os.path.dirname(os.path.realpath(__file__)) sub_folder_dir = os.path.join(current_file_dir, "mellanox") if sub_folder_dir not in sys.path: sys.path.append(sub_folder_dir) from check_hw_mgmt_service import check_hw_management_service from check_sysfs import check_sysfs logging.info("Check the hw-management service") check_hw_management_service(dut) logging.info("Check sysfs") check_sysfs(dut)
def test_reload_configuration(localhost, ansible_adhoc, testbed): """ @summary: This test case is to reload the configuration and check platform status """ hostname = testbed['dut'] ans_host = AnsibleHost(ansible_adhoc, hostname) ans_host.command("show platform summary") lab_conn_graph_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), \ "../../ansible/files/lab_connection_graph.xml") conn_graph_facts = localhost.conn_graph_facts(host=hostname, filename=lab_conn_graph_file).\ contacted['localhost']['ansible_facts'] interfaces = conn_graph_facts["device_conn"] asic_type = ans_host.shell( "show platform summary | awk '/ASIC: / {print$2}'")["stdout"].strip() logging.info("Reload configuration") ans_host.command("sudo config reload -y") logging.info("Wait until all critical services are fully started") check_critical_services(ans_host) logging.info("Wait some time for all the transceivers to be detected") assert wait_until(300, 20, all_transceivers_detected, ans_host, interfaces), \ "Not all transceivers are detected in 300 seconds" logging.info("Check interface status") time.sleep(60) check_interface_status(ans_host, interfaces) logging.info("Check transceiver status") check_transceiver_basic(ans_host, interfaces) if asic_type in ["mellanox"]: current_file_dir = os.path.dirname(os.path.realpath(__file__)) sub_folder_dir = os.path.join(current_file_dir, "mellanox") if sub_folder_dir not in sys.path: sys.path.append(sub_folder_dir) from check_hw_mgmt_service import check_hw_management_service from check_sysfs import check_sysfs logging.info("Check the hw-management service") check_hw_management_service(ans_host) logging.info("Check sysfs") check_sysfs(ans_host)
def restart_service_and_check(localhost, dut, service): """ Restart specified service and check platform status """ dut.command("show platform summary") lab_conn_graph_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), \ "../../ansible/files/lab_connection_graph.xml") conn_graph_facts = localhost.conn_graph_facts(host=dut.hostname, filename=lab_conn_graph_file).\ contacted['localhost']['ansible_facts'] interfaces = conn_graph_facts["device_conn"] asic_type = dut.shell( "show platform summary | awk '/ASIC: / {print$2}'")["stdout"].strip() logging.info("Restart the %s service" % service) dut.command("sudo systemctl restart %s" % service) logging.info("Wait until all critical services are fully started") check_critical_services(dut) logging.info("Wait some time for all the transceivers to be detected") assert wait_until(300, 20, all_transceivers_detected, dut, interfaces), \ "Not all transceivers are detected in 300 seconds" logging.info("Check interface status") time.sleep(60) check_interface_status(dut, interfaces) logging.info("Check transceiver status") check_transceiver_basic(dut, interfaces) if asic_type in ["mellanox"]: current_file_dir = os.path.dirname(os.path.realpath(__file__)) sub_folder_dir = os.path.join(current_file_dir, "mellanox") if sub_folder_dir not in sys.path: sys.path.append(sub_folder_dir) from check_hw_mgmt_service import check_hw_management_service from check_sysfs import check_sysfs logging.info("Check the hw-management service") check_hw_management_service(dut) logging.info("Check sysfs") check_sysfs(dut)
def reboot_and_check(localhost, dut, interfaces, reboot_type=REBOOT_TYPE_COLD, reboot_helper=None, reboot_kwargs=None): """ Perform the specified type of reboot and check platform status. @param localhost: The Localhost object. @param dut: The AnsibleHost object of DUT. @param interfaces: DUT's interfaces defined by minigraph @param reboot_type: The reboot type, pre-defined const that has name convention of REBOOT_TYPE_XXX. @param reboot_helper: The helper function used only by power off reboot @param reboot_kwargs: The argument used by reboot_helper """ logging.info("Run %s reboot on DUT" % reboot_type) assert reboot_type in reboot_ctrl_dict.keys( ), "Unknown reboot type %s" % reboot_type reboot_timeout = reboot_ctrl_dict[reboot_type]["timeout"] reboot_cause = reboot_ctrl_dict[reboot_type]["cause"] dut_datetime = datetime.strptime( dut.command('date -u +"%Y-%m-%d %H:%M:%S"')["stdout"], "%Y-%m-%d %H:%M:%S") if reboot_type == REBOOT_TYPE_POWEROFF: assert reboot_helper is not None, "A reboot function must be provided for power off reboot" reboot_helper(reboot_kwargs) localhost.wait_for(host=dut.hostname, port=22, state="stopped", delay=10, timeout=120) else: reboot_cmd = reboot_ctrl_dict[reboot_type]["command"] reboot_task, reboot_res = dut.command(reboot_cmd, module_ignore_errors=True, module_async=True) logging.info("Wait for DUT to go down") res = localhost.wait_for(host=dut.hostname, port=22, state="stopped", timeout=180, module_ignore_errors=True) if "failed" in res: try: logging.error( "Wait for switch down failed, try to kill any possible stuck reboot task" ) pid = dut.command("pgrep -f '%s'" % reboot_cmd)["stdout"] dut.command("kill -9 %s" % pid) reboot_task.terminate() logging.error("Result of command '%s': " + str(reboot_res.get(timeout=0))) except Exception as e: logging.error( "Exception raised while cleanup reboot task and get result: " + repr(e)) logging.info("Wait for DUT to come back") localhost.wait_for(host=dut.hostname, port=22, state="started", delay=10, timeout=reboot_timeout) logging.info("Check the uptime to verify whether reboot was performed") dut_uptime = datetime.strptime( dut.command("uptime -s")["stdout"], "%Y-%m-%d %H:%M:%S") assert float(dut_uptime.strftime("%s")) - float( dut_datetime.strftime("%s")) > 10, "Device did not reboot" logging.info("Wait until all critical services are fully started") check_critical_services(dut) logging.info("Check reboot cause") check_reboot_cause(dut, reboot_cause) if reboot_ctrl_dict[reboot_type]["test_reboot_cause_only"]: logging.info( "Further checking skipped for %s test which intends to verify reboot-cause only" .format(reboot_type)) return logging.info("Wait some time for all the transceivers to be detected") assert wait_until(300, 20, check_interface_information, dut, interfaces), \ "Not all transceivers are detected or interfaces are up in 300 seconds" logging.info("Check transceiver status") check_transceiver_basic(dut, interfaces) logging.info("Check pmon daemon status") assert check_pmon_daemon_status(dut), "Not all pmon daemons running." if dut.facts["asic_type"] in ["mellanox"]: current_file_dir = os.path.dirname(os.path.realpath(__file__)) sub_folder_dir = os.path.join(current_file_dir, "mellanox") if sub_folder_dir not in sys.path: sys.path.append(sub_folder_dir) from check_hw_mgmt_service import check_hw_management_service from check_sysfs import check_sysfs logging.info("Check the hw-management service") check_hw_management_service(dut) logging.info("Check sysfs") check_sysfs(dut)
def reboot_and_check(localhost, dut, reboot_type="cold"): """ Perform the specified type of reboot and check platform status. """ dut.command("show platform summary") lab_conn_graph_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), \ "../../ansible/files/lab_connection_graph.xml") conn_graph_facts = localhost.conn_graph_facts(host=dut.hostname, filename=lab_conn_graph_file).\ contacted['localhost']['ansible_facts'] interfaces = conn_graph_facts["device_conn"] asic_type = dut.shell("show platform summary | awk '/ASIC: / {print$2}'")["stdout"].strip() logging.info("Run %s reboot on DUT" % reboot_type) if reboot_type == "cold": reboot_cmd = "sudo reboot &" reboot_timeout = 300 elif reboot_type == "fast": reboot_cmd = "sudo fast-reboot &" reboot_timeout = 180 elif reboot_type == "warm": reboot_cmd = "sudo warm-reboot &" reboot_timeout = 180 else: assert False, "Reboot type %s is not supported" % reboot_type dut.shell(reboot_cmd) logging.info("Wait for DUT to go down") localhost.wait_for(host=dut.hostname, port=22, state="stopped", delay=10, timeout=120) logging.info("Wait for DUT to come back") localhost.wait_for(host=dut.hostname, port=22, state="started", delay=10, timeout=reboot_timeout) logging.info("Wait until all critical services are fully started") check_critical_services(dut) logging.info("Wait some time for all the transceivers to be detected") assert wait_until(300, 20, all_transceivers_detected, dut, interfaces), \ "Not all transceivers are detected in 300 seconds" logging.info("Check interface status") check_interface_status(dut, interfaces) logging.info("Check transceiver status") check_transceiver_basic(dut, interfaces) if asic_type in ["mellanox"]: current_file_dir = os.path.dirname(os.path.realpath(__file__)) sub_folder_dir = os.path.join(current_file_dir, "mellanox") if sub_folder_dir not in sys.path: sys.path.append(sub_folder_dir) from check_hw_mgmt_service import check_hw_management_service from check_hw_mgmt_service import wait_until_fan_speed_set_to_default from check_sysfs import check_sysfs logging.info("Wait until fan speed is set to default") wait_until_fan_speed_set_to_default(dut) logging.info("Check the hw-management service") check_hw_management_service(dut) logging.info("Check sysfs") check_sysfs(dut)
def reboot_and_check(localhost, dut, interfaces, reboot_type=REBOOT_TYPE_COLD, reboot_helper=None, reboot_kwargs=None): """ Perform the specified type of reboot and check platform status. @param dut: The AnsibleHost object of DUT. @param interfaces: DUT's interfaces defined by minigraph @param reboot_type: The reboot type, pre-defined const that has name convention of REBOOT_TYPE_XXX. @param reboot_helper: The helper function used only by power off reboot @param reboot_kwargs: The argument used by reboot_helper """ logging.info("Run %s reboot on DUT" % reboot_type) assert reboot_type in reboot_ctrl_dict.keys( ), "Unknown reboot type %s" % reboot_type reboot_timeout = reboot_ctrl_dict[reboot_type]["timeout"] reboot_cause = reboot_ctrl_dict[reboot_type]["cause"] if reboot_type == REBOOT_TYPE_POWEROFF: assert reboot_helper is not None, "A reboot function must be provided for power off reboot" reboot_helper(reboot_kwargs) localhost.wait_for(host=dut.hostname, port=22, state="stopped", delay=10, timeout=120) else: reboot_cmd = reboot_ctrl_dict[reboot_type]["command"] process, queue = dut.command(reboot_cmd, module_async=True) logging.info("Wait for DUT to go down") res = localhost.wait_for(host=dut.hostname, port=22, state="stopped", delay=10, timeout=120, module_ignore_errors=True) if "failed" in res: if process.is_alive(): logging.error("Command '%s' is not completed" % reboot_cmd) process.terminate() logging.error("reboot result %s" % str(queue.get())) assert False, "DUT did not go down" logging.info("Wait for DUT to come back") localhost.wait_for(host=dut.hostname, port=22, state="started", delay=10, timeout=reboot_timeout) logging.info("Wait until all critical services are fully started") check_critical_services(dut) logging.info("Check reboot cause") check_reboot_cause(dut, reboot_cause) logging.info("Wait some time for all the transceivers to be detected") assert wait_until(300, 20, check_interface_information, dut, interfaces), \ "Not all transceivers are detected or interfaces are up in 300 seconds" logging.info("Check transceiver status") check_transceiver_basic(dut, interfaces) logging.info("Check pmon daemon status") assert check_pmon_daemon_status(dut), "Not all pmon daemons running." if dut.facts["asic_type"] in ["mellanox"]: current_file_dir = os.path.dirname(os.path.realpath(__file__)) sub_folder_dir = os.path.join(current_file_dir, "mellanox") if sub_folder_dir not in sys.path: sys.path.append(sub_folder_dir) from check_hw_mgmt_service import check_hw_management_service from check_sysfs import check_sysfs logging.info("Check the hw-management service") check_hw_management_service(dut) logging.info("Check sysfs") check_sysfs(dut)