def reboot_dut(dut, localhost, cmd, wait_time): logger.info("Reboot dut using cmd='%s'" % cmd) reboot_task, reboot_res = dut.command(cmd, module_async=True) logger.info("Wait for DUT to go down") try: localhost.wait_for(host=dut.hostname, port=22, state="stopped", delay=10, timeout=300) except RunAnsibleModuleFail as e: logger.error("DUT did not go down, exception: " + repr(e)) if reboot_task.is_alive(): logger.error("Rebooting is not completed") reboot_task.terminate() logger.error("reboot result %s" % str(reboot_res.get())) assert False, "Failed to reboot the DUT" localhost.wait_for(host=dut.hostname, port=22, state="started", delay=10, timeout=300) wait(wait_time, msg="Wait {} seconds for system to be stable.".format(wait_time))
def test_config_interface_ip(self, setup_config_mode, sample_intf): """ Checks whether 'config interface ip add/remove <intf> <ip>' adds/removes the ip on the test interface when its interface alias/name is provided as per the configured naming mode """ dutHostGuest, mode, ifmode = setup_config_mode test_intf = sample_intf[mode] out = dutHostGuest.shell('SONIC_CLI_IFACE_MODE={} sudo config interface ip remove {} 10.0.0.0/31'.format(ifmode, test_intf)) if out['rc'] != 0: pytest.fail() wait(3) show_ip_intf = dutHostGuest.shell('SONIC_CLI_IFACE_MODE={} show ip interface'.format(ifmode))['stdout'] logger.info('show_ip_intf:\n{}'.format(show_ip_intf)) assert re.search(r'{}\s+10.0.0.0/31'.format(test_intf), show_ip_intf) is None out = dutHostGuest.shell('SONIC_CLI_IFACE_MODE={} sudo config interface ip add {} 10.0.0.0/31'.format(ifmode, test_intf)) if out['rc'] != 0: pytest.fail() wait(3) show_ip_intf = dutHostGuest.shell('SONIC_CLI_IFACE_MODE={} show ip interface'.format(ifmode))['stdout'] logger.info('show_ip_intf:\n{}'.format(show_ip_intf)) assert re.search(r'{}\s+10.0.0.0/31'.format(test_intf), show_ip_intf) is not None
def check_services(dut): logger.info("Checking services status...") networking_uptime = dut.get_networking_uptime().seconds timeout = max((SYSTEM_STABILIZE_MAX_TIME - networking_uptime), 0) interval = 20 logger.info("networking_uptime=%d seconds, timeout=%d seconds, interval=%d seconds" % \ (networking_uptime, timeout, interval)) check_result = {"failed": True, "check_item": "services"} if timeout == 0: # Check services status, do not retry. services_status = dut.critical_services_status() check_result["failed"] = False if all( services_status.values()) else True check_result["services_status"] = services_status else: # Retry checking service status start = time.time() elapsed = 0 while elapsed < timeout: services_status = dut.critical_services_status() check_result["failed"] = False if all( services_status.values()) else True check_result["services_status"] = services_status if check_result["failed"]: wait(interval, msg="Not all services are started, wait %d seconds to retry. Remaining time: %d %s" % \ (interval, int(timeout - elapsed), str(check_result["services_status"]))) elapsed = time.time() - start else: break logger.info("Done checking services status.") return check_result
def __recover_interfaces(dut, fanouthosts, result, wait_time): for port in result['down_ports']: logging.info("Restoring port {}".format(port)) fanout, fanout_port = fanout_switch_port_lookup(fanouthosts, port) if fanout and fanout_port: fanout.no_shutdown(fanout_port) dut.no_shutdown(port) wait(wait_time, msg="Wait {} seconds for interface(s) to restore.".format(wait_time))
def recover(dut, localhost, recover_method): logger.info("Try to recover %s using method %s" % (dut.hostname, recover_method)) if constants.RECOVER_METHODS[recover_method]["reboot"]: reboot_dut(dut, localhost, constants.RECOVER_METHODS[recover_method]["cmd"]) else: dut.command(constants.RECOVER_METHODS[recover_method]["cmd"]) wait(30, msg="Wait 30 seconds for system to be stable.")
def __recover_interfaces(dut, fanouthosts, result, wait_time): action = None for port in result['down_ports']: logging.info("Restoring port {}".format(port)) pn = str(port).lower() if 'portchannel' in pn or 'vlan' in pn: action = 'config_reload' continue fanout, fanout_port = fanout_switch_port_lookup(fanouthosts, port) if fanout and fanout_port: fanout.no_shutdown(fanout_port) dut.no_shutdown(port) wait(wait_time, msg="Wait {} seconds for interface(s) to restore.".format(wait_time)) return action
def check_interfaces(dut): logger.info("Checking interfaces status...") networking_uptime = dut.get_networking_uptime().seconds timeout = max((SYSTEM_STABILIZE_MAX_TIME - networking_uptime), 0) interval = 20 logger.info("networking_uptime=%d seconds, timeout=%d seconds, interval=%d seconds" % \ (networking_uptime, timeout, interval)) cfg_facts = dut.config_facts(host=dut.hostname, source="persistent")['ansible_facts'] interfaces = [ k for k, v in cfg_facts["PORT"].items() if "admin_status" in v and v["admin_status"] == "up" ] if "PORTCHANNEL_INTERFACE" in cfg_facts: interfaces += cfg_facts["PORTCHANNEL_INTERFACE"].keys() if "VLAN_INTERFACE" in cfg_facts: interfaces += cfg_facts["VLAN_INTERFACE"].keys() logger.info(json.dumps(interfaces, indent=4)) check_result = {"failed": True, "check_item": "interfaces"} if timeout == 0: # Check interfaces status, do not retry. down_ports = _find_down_ports(dut, interfaces) check_result["failed"] = True if len(down_ports) > 0 else False check_result["down_ports"] = down_ports else: # Retry checking interface status start = time.time() elapsed = 0 while elapsed < timeout: down_ports = _find_down_ports(dut, interfaces) check_result["failed"] = True if len(down_ports) > 0 else False check_result["down_ports"] = down_ports if check_result["failed"]: wait(interval, msg="Found down ports, wait %d seconds to retry. Remaining time: %d, down_ports=%s" % \ (interval, int(timeout - elapsed), str(check_result["down_ports"]))) elapsed = time.time() - start else: break logger.info("Done checking interfaces status.") return check_result
def test_config_interface_state(self, setup_config_mode, sample_intf): """ Checks whether 'config interface startup/shutdown <intf>' changes the admin state of the test interface to up/down when its interface alias/name is provided as per the configured naming mode """ dutHostGuest, mode, ifmode = setup_config_mode test_intf = sample_intf[mode] interface = sample_intf['default'] regex_int = re.compile(r'(\S+)\s+[\d,N\/A]+\s+(\w+)\s+(\d+)\s+([\w\/]+)\s+(\w+)\s+(\w+)\s+(\w+)') out = dutHostGuest.shell('SONIC_CLI_IFACE_MODE={} sudo config interface shutdown {}'.format(ifmode, test_intf)) if out['rc'] != 0: pytest.fail() wait(3) show_intf_status = dutHostGuest.shell('SONIC_CLI_IFACE_MODE={0} show interfaces status {1} | grep -w {1}'.format(ifmode, test_intf)) logger.info('show_intf_status:\n{}'.format(show_intf_status['stdout'])) line = show_intf_status['stdout'].strip() if regex_int.match(line) and interface == regex_int.match(line).group(1): admin_state = regex_int.match(line).group(7) assert admin_state == 'down' out = dutHostGuest.shell('SONIC_CLI_IFACE_MODE={} sudo config interface startup {}'.format(ifmode, test_intf)) if out['rc'] != 0: pytest.fail() wait(3) show_intf_status = dutHostGuest.shell('SONIC_CLI_IFACE_MODE={0} show interfaces status {1} | grep -w {1}'.format(ifmode, test_intf)) logger.info('show_intf_status:\n{}'.format(show_intf_status['stdout'])) line = show_intf_status['stdout'].strip() if regex_int.match(line) and interface == regex_int.match(line).group(1): admin_state = regex_int.match(line).group(7) assert admin_state == 'up'
def test_portstat_clear(duthost, command): before_portstat = parse_portstat( duthost.command('portstat')['stdout_lines']) pytest_assert(before_portstat, 'No parsed command output') duthost.command(command) wait(5, 'Wait for portstat counters to refresh') after_portstat = parse_portstat( duthost.command('portstat')['stdout_lines']) pytest_assert(after_portstat, 'No parsed command output') for intf in before_portstat: pytest_assert( int(before_portstat[intf]['rx_ok']) >= int( after_portstat[intf]['rx_ok']), 'Value of RX_OK after clear should be lesser') pytest_assert( int(before_portstat[intf]['tx_ok']) >= int( after_portstat[intf]['rx_ok']), 'Value of RX_OK after clear should be lesser')
def __recover_with_command(dut, cmd, wait_time): dut.command(cmd) wait(wait_time, msg="Wait {} seconds for system to be stable.".format(wait_time))